]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/gpu/drm/radeon/r600.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: add missing ttm_eu_backoff_reservation to radeon_bo_list_validate
[mirror_ubuntu-zesty-kernel.git] / drivers / gpu / drm / radeon / r600.c
CommitLineData
771fe6b9
JG		 1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
5a0e3ad6 28#include <linux/slab.h>
3ce0a23d
JG		 29#include <linux/seq_file.h>
30#include <linux/firmware.h>
31#include <linux/platform_device.h>
e0cd3608 32#include <linux/module.h>
760285e7
DH		 33#include <drm/drmP.h>
34#include <drm/radeon_drm.h>
771fe6b9 35#include "radeon.h"
e6990375 36#include "radeon_asic.h"
3ce0a23d 37#include "radeon_mode.h"
3ce0a23d 38#include "r600d.h"
3ce0a23d 39#include "atom.h"
d39c3b89 40#include "avivod.h"
138e4e16 41#include "radeon_ucode.h"
3ce0a23d
JG		 42
43/* Firmware Names */
44MODULE_FIRMWARE("radeon/R600_pfp.bin");
45MODULE_FIRMWARE("radeon/R600_me.bin");
46MODULE_FIRMWARE("radeon/RV610_pfp.bin");
47MODULE_FIRMWARE("radeon/RV610_me.bin");
48MODULE_FIRMWARE("radeon/RV630_pfp.bin");
49MODULE_FIRMWARE("radeon/RV630_me.bin");
50MODULE_FIRMWARE("radeon/RV620_pfp.bin");
51MODULE_FIRMWARE("radeon/RV620_me.bin");
52MODULE_FIRMWARE("radeon/RV635_pfp.bin");
53MODULE_FIRMWARE("radeon/RV635_me.bin");
54MODULE_FIRMWARE("radeon/RV670_pfp.bin");
55MODULE_FIRMWARE("radeon/RV670_me.bin");
56MODULE_FIRMWARE("radeon/RS780_pfp.bin");
57MODULE_FIRMWARE("radeon/RS780_me.bin");
58MODULE_FIRMWARE("radeon/RV770_pfp.bin");
59MODULE_FIRMWARE("radeon/RV770_me.bin");
66229b20 60MODULE_FIRMWARE("radeon/RV770_smc.bin");
3ce0a23d
JG		 61MODULE_FIRMWARE("radeon/RV730_pfp.bin");
62MODULE_FIRMWARE("radeon/RV730_me.bin");
66229b20
AD		 63MODULE_FIRMWARE("radeon/RV730_smc.bin");
64MODULE_FIRMWARE("radeon/RV740_smc.bin");
3ce0a23d
JG		 65MODULE_FIRMWARE("radeon/RV710_pfp.bin");
66MODULE_FIRMWARE("radeon/RV710_me.bin");
66229b20 67MODULE_FIRMWARE("radeon/RV710_smc.bin");
d8f60cfc
AD		 68MODULE_FIRMWARE("radeon/R600_rlc.bin");
69MODULE_FIRMWARE("radeon/R700_rlc.bin");
fe251e2f
AD		 70MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
71MODULE_FIRMWARE("radeon/CEDAR_me.bin");
45f9a39b 72MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
dc50ba7f 73MODULE_FIRMWARE("radeon/CEDAR_smc.bin");
fe251e2f
AD		 74MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
75MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
45f9a39b 76MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
dc50ba7f 77MODULE_FIRMWARE("radeon/REDWOOD_smc.bin");
fe251e2f
AD		 78MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
79MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
45f9a39b 80MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
dc50ba7f 81MODULE_FIRMWARE("radeon/JUNIPER_smc.bin");
a7433742 82MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
fe251e2f 83MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
45f9a39b 84MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
dc50ba7f 85MODULE_FIRMWARE("radeon/CYPRESS_smc.bin");
439bd6cd
AD		 86MODULE_FIRMWARE("radeon/PALM_pfp.bin");
87MODULE_FIRMWARE("radeon/PALM_me.bin");
88MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
d5c5a72f
AD		 89MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
90MODULE_FIRMWARE("radeon/SUMO_me.bin");
91MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
92MODULE_FIRMWARE("radeon/SUMO2_me.bin");
3ce0a23d 93
f13f7731
AD		 94static const u32 crtc_offsets[2] =
95{
96 0,
97 AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL
98};
99
3ce0a23d 100int r600_debugfs_mc_info_init(struct radeon_device *rdev);
771fe6b9 101
1a029b76 102/* r600,rv610,rv630,rv620,rv635,rv670 */
771fe6b9 103int r600_mc_wait_for_idle(struct radeon_device *rdev);
1109ca09 104static void r600_gpu_init(struct radeon_device *rdev);
3ce0a23d 105void r600_fini(struct radeon_device *rdev);
45f9a39b 106void r600_irq_disable(struct radeon_device *rdev);
9e46a48d 107static void r600_pcie_gen2_enable(struct radeon_device *rdev);
2948f5e6 108extern int evergreen_rlc_resume(struct radeon_device *rdev);
771fe6b9 109
454d2e2a
AD		 110/**
111 * r600_get_xclk - get the xclk
112 *
113 * @rdev: radeon_device pointer
114 *
115 * Returns the reference clock used by the gfx engine
116 * (r6xx, IGPs, APUs).
117 */
118u32 r600_get_xclk(struct radeon_device *rdev)
119{
120 return rdev->clock.spll.reference_freq;
121}
122
21a8122a 123/* get temperature in millidegrees */
20d391d7 124int rv6xx_get_temp(struct radeon_device *rdev)
21a8122a
AD		 125{
126 u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
127 ASIC_T_SHIFT;
20d391d7 128 int actual_temp = temp & 0xff;
21a8122a 129
20d391d7
AD		 130 if (temp & 0x100)
131 actual_temp -= 256;
132
133 return actual_temp * 1000;
21a8122a
AD		 134}
135
ce8f5370 136void r600_pm_get_dynpm_state(struct radeon_device *rdev)
a48b9b4e
AD		 137{
138 int i;
139
ce8f5370
AD		 140 rdev->pm.dynpm_can_upclock = true;
141 rdev->pm.dynpm_can_downclock = true;
a48b9b4e
AD		 142
143 /* power state array is low to high, default is first */
144 if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {
145 int min_power_state_index = 0;
146
147 if (rdev->pm.num_power_states > 2)
148 min_power_state_index = 1;
149
ce8f5370
AD		 150 switch (rdev->pm.dynpm_planned_action) {
151 case DYNPM_ACTION_MINIMUM:
a48b9b4e
AD		 152 rdev->pm.requested_power_state_index = min_power_state_index;
153 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 154 rdev->pm.dynpm_can_downclock = false;
a48b9b4e 155 break;
ce8f5370 156 case DYNPM_ACTION_DOWNCLOCK:
a48b9b4e
AD		 157 if (rdev->pm.current_power_state_index == min_power_state_index) {
158 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
ce8f5370 159 rdev->pm.dynpm_can_downclock = false;
a48b9b4e
AD		 160 } else {
161 if (rdev->pm.active_crtc_count > 1) {
162 for (i = 0; i < rdev->pm.num_power_states; i++) {
d7311171 163 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
a48b9b4e
AD		 164 continue;
165 else if (i >= rdev->pm.current_power_state_index) {
166 rdev->pm.requested_power_state_index =
167 rdev->pm.current_power_state_index;
168 break;
169 } else {
170 rdev->pm.requested_power_state_index = i;
171 break;
172 }
173 }
773c3fa3
AD		 174 } else {
175 if (rdev->pm.current_power_state_index == 0)
176 rdev->pm.requested_power_state_index =
177 rdev->pm.num_power_states - 1;
178 else
179 rdev->pm.requested_power_state_index =
180 rdev->pm.current_power_state_index - 1;
181 }
a48b9b4e
AD		 182 }
183 rdev->pm.requested_clock_mode_index = 0;
d7311171
AD		 184 /* don't use the power state if crtcs are active and no display flag is set */
185 if ((rdev->pm.active_crtc_count > 0) &&
186 (rdev->pm.power_state[rdev->pm.requested_power_state_index].
187 clock_info[rdev->pm.requested_clock_mode_index].flags &
188 RADEON_PM_MODE_NO_DISPLAY)) {
189 rdev->pm.requested_power_state_index++;
190 }
a48b9b4e 191 break;
ce8f5370 192 case DYNPM_ACTION_UPCLOCK:
a48b9b4e
AD		 193 if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
194 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
ce8f5370 195 rdev->pm.dynpm_can_upclock = false;
a48b9b4e
AD		 196 } else {
197 if (rdev->pm.active_crtc_count > 1) {
198 for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
d7311171 199 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
a48b9b4e
AD		 200 continue;
201 else if (i <= rdev->pm.current_power_state_index) {
202 rdev->pm.requested_power_state_index =
203 rdev->pm.current_power_state_index;
204 break;
205 } else {
206 rdev->pm.requested_power_state_index = i;
207 break;
208 }
209 }
210 } else
211 rdev->pm.requested_power_state_index =
212 rdev->pm.current_power_state_index + 1;
213 }
214 rdev->pm.requested_clock_mode_index = 0;
215 break;
ce8f5370 216 case DYNPM_ACTION_DEFAULT:
58e21dff
AD		 217 rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
218 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 219 rdev->pm.dynpm_can_upclock = false;
58e21dff 220 break;
ce8f5370 221 case DYNPM_ACTION_NONE:
a48b9b4e
AD		 222 default:
223 DRM_ERROR("Requested mode for not defined action\n");
224 return;
225 }
226 } else {
227 /* XXX select a power state based on AC/DC, single/dualhead, etc. */
228 /* for now just select the first power state and switch between clock modes */
229 /* power state array is low to high, default is first (0) */
230 if (rdev->pm.active_crtc_count > 1) {
231 rdev->pm.requested_power_state_index = -1;
232 /* start at 1 as we don't want the default mode */
233 for (i = 1; i < rdev->pm.num_power_states; i++) {
d7311171 234 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
a48b9b4e
AD		 235 continue;
236 else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||
237 (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
238 rdev->pm.requested_power_state_index = i;
239 break;
240 }
241 }
242 /* if nothing selected, grab the default state. */
243 if (rdev->pm.requested_power_state_index == -1)
244 rdev->pm.requested_power_state_index = 0;
245 } else
246 rdev->pm.requested_power_state_index = 1;
247
ce8f5370
AD		 248 switch (rdev->pm.dynpm_planned_action) {
249 case DYNPM_ACTION_MINIMUM:
a48b9b4e 250 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 251 rdev->pm.dynpm_can_downclock = false;
a48b9b4e 252 break;
ce8f5370 253 case DYNPM_ACTION_DOWNCLOCK:
a48b9b4e
AD		 254 if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
255 if (rdev->pm.current_clock_mode_index == 0) {
256 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 257 rdev->pm.dynpm_can_downclock = false;
a48b9b4e
AD		 258 } else
259 rdev->pm.requested_clock_mode_index =
260 rdev->pm.current_clock_mode_index - 1;
261 } else {
262 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 263 rdev->pm.dynpm_can_downclock = false;
a48b9b4e 264 }
d7311171
AD		 265 /* don't use the power state if crtcs are active and no display flag is set */
266 if ((rdev->pm.active_crtc_count > 0) &&
267 (rdev->pm.power_state[rdev->pm.requested_power_state_index].
268 clock_info[rdev->pm.requested_clock_mode_index].flags &
269 RADEON_PM_MODE_NO_DISPLAY)) {
270 rdev->pm.requested_clock_mode_index++;
271 }
a48b9b4e 272 break;
ce8f5370 273 case DYNPM_ACTION_UPCLOCK:
a48b9b4e
AD		 274 if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
275 if (rdev->pm.current_clock_mode_index ==
276 (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
277 rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
ce8f5370 278 rdev->pm.dynpm_can_upclock = false;
a48b9b4e
AD		 279 } else
280 rdev->pm.requested_clock_mode_index =
281 rdev->pm.current_clock_mode_index + 1;
282 } else {
283 rdev->pm.requested_clock_mode_index =
284 rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
ce8f5370 285 rdev->pm.dynpm_can_upclock = false;
a48b9b4e
AD		 286 }
287 break;
ce8f5370 288 case DYNPM_ACTION_DEFAULT:
58e21dff
AD		 289 rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
290 rdev->pm.requested_clock_mode_index = 0;
ce8f5370 291 rdev->pm.dynpm_can_upclock = false;
58e21dff 292 break;
ce8f5370 293 case DYNPM_ACTION_NONE:
a48b9b4e
AD		 294 default:
295 DRM_ERROR("Requested mode for not defined action\n");
296 return;
297 }
298 }
299
d9fdaafb 300 DRM_DEBUG_DRIVER("Requested: e: %d m: %d p: %d\n",
ce8a3eb2
AD		 301 rdev->pm.power_state[rdev->pm.requested_power_state_index].
302 clock_info[rdev->pm.requested_clock_mode_index].sclk,
303 rdev->pm.power_state[rdev->pm.requested_power_state_index].
304 clock_info[rdev->pm.requested_clock_mode_index].mclk,
305 rdev->pm.power_state[rdev->pm.requested_power_state_index].
306 pcie_lanes);
a48b9b4e
AD		 307}
308
ce8f5370
AD		 309void rs780_pm_init_profile(struct radeon_device *rdev)
310{
311 if (rdev->pm.num_power_states == 2) {
312 /* default */
313 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
314 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
315 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
316 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
317 /* low sh */
318 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 0;
319 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 0;
320 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
321 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 322 /* mid sh */
323 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 0;
324 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 0;
325 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
326 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 327 /* high sh */
328 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 0;
329 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
330 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
331 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
332 /* low mh */
333 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 0;
334 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
335 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
336 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 337 /* mid mh */
338 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 0;
339 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
340 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
341 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 342 /* high mh */
343 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 0;
344 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 1;
345 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
346 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
347 } else if (rdev->pm.num_power_states == 3) {
348 /* default */
349 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
350 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
351 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
352 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
353 /* low sh */
354 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
355 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
356 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
357 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 358 /* mid sh */
359 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
360 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
361 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
362 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 363 /* high sh */
364 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
365 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 2;
366 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
367 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
368 /* low mh */
369 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 1;
370 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 1;
371 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
372 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 373 /* mid mh */
374 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 1;
375 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 1;
376 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
377 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 378 /* high mh */
379 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 1;
380 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
381 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
382 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
383 } else {
384 /* default */
385 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
386 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
387 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
388 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
389 /* low sh */
390 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 2;
391 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 2;
392 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
393 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 394 /* mid sh */
395 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 2;
396 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 2;
397 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
398 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 399 /* high sh */
400 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 2;
401 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 3;
402 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
403 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
404 /* low mh */
405 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
406 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
407 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
408 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 409 /* mid mh */
410 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
411 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
412 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
413 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 414 /* high mh */
415 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
416 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 3;
417 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
418 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
419 }
420}
bae6b562 421
ce8f5370
AD		 422void r600_pm_init_profile(struct radeon_device *rdev)
423{
bbe26ffe
AD		 424 int idx;
425
ce8f5370
AD		 426 if (rdev->family == CHIP_R600) {
427 /* XXX */
428 /* default */
429 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
430 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
431 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
4bff5171 432 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 433 /* low sh */
434 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
435 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
436 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
4bff5171 437 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 438 /* mid sh */
439 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
440 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
441 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
442 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 443 /* high sh */
444 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
445 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
446 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
4bff5171 447 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 448 /* low mh */
449 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
450 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
451 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
4bff5171 452 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
c9e75b21
AD		 453 /* mid mh */
454 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
455 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
456 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
457 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 458 /* high mh */
459 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
460 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
461 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
4bff5171 462 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
ce8f5370
AD		 463 } else {
464 if (rdev->pm.num_power_states < 4) {
465 /* default */
466 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
467 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
468 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
469 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
470 /* low sh */
4bff5171
AD		 471 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
472 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
ce8f5370 473 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
c9e75b21
AD		 474 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
475 /* mid sh */
476 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
477 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
478 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
479 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
ce8f5370 480 /* high sh */
4bff5171
AD		 481 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
482 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
ce8f5370
AD		 483 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
484 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
485 /* low mh */
4bff5171
AD		 486 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
487 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 2;
ce8f5370 488 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
c9e75b21
AD		 489 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
490 /* low mh */
491 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
492 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 2;
493 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
494 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
ce8f5370 495 /* high mh */
4bff5171
AD		 496 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
497 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
ce8f5370
AD		 498 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
499 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
500 } else {
501 /* default */
502 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
503 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
504 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
505 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
506 /* low sh */
bbe26ffe
AD		 507 if (rdev->flags & RADEON_IS_MOBILITY)
508 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
509 else
510 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
511 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
512 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
513 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
514 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
c9e75b21 515 /* mid sh */
bbe26ffe
AD		 516 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
517 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
518 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
519 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
ce8f5370 520 /* high sh */
bbe26ffe
AD		 521 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
522 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
523 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
ce8f5370
AD		 524 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
525 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
526 /* low mh */
bbe26ffe
AD		 527 if (rdev->flags & RADEON_IS_MOBILITY)
528 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 1);
529 else
530 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
531 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
532 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
533 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
534 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
c9e75b21 535 /* mid mh */
bbe26ffe
AD		 536 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
537 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
538 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
539 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
ce8f5370 540 /* high mh */
bbe26ffe
AD		 541 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
542 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
543 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
ce8f5370
AD		 544 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
545 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
546 }
547 }
bae6b562
AD		 548}
549
49e02b73
AD		 550void r600_pm_misc(struct radeon_device *rdev)
551{
a081a9d6
RM		 552 int req_ps_idx = rdev->pm.requested_power_state_index;
553 int req_cm_idx = rdev->pm.requested_clock_mode_index;
554 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
555 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
7ac9aa5a 556
4d60173f 557 if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
a377e187
AD		 558 /* 0xff01 is a flag rather then an actual voltage */
559 if (voltage->voltage == 0xff01)
560 return;
4d60173f 561 if (voltage->voltage != rdev->pm.current_vddc) {
8a83ec5e 562 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 563 rdev->pm.current_vddc = voltage->voltage;
d9fdaafb 564 DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
4d60173f
AD		 565 }
566 }
49e02b73
AD		 567}
568
def9ba9c
AD		 569bool r600_gui_idle(struct radeon_device *rdev)
570{
571 if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
572 return false;
573 else
574 return true;
575}
576
e0df1ac5
AD		 577/* hpd for digital panel detect/disconnect */
578bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
579{
580 bool connected = false;
581
582 if (ASIC_IS_DCE3(rdev)) {
583 switch (hpd) {
584 case RADEON_HPD_1:
585 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
586 connected = true;
587 break;
588 case RADEON_HPD_2:
589 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
590 connected = true;
591 break;
592 case RADEON_HPD_3:
593 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
594 connected = true;
595 break;
596 case RADEON_HPD_4:
597 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
598 connected = true;
599 break;
600 /* DCE 3.2 */
601 case RADEON_HPD_5:
602 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
603 connected = true;
604 break;
605 case RADEON_HPD_6:
606 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
607 connected = true;
608 break;
609 default:
610 break;
611 }
612 } else {
613 switch (hpd) {
614 case RADEON_HPD_1:
615 if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
616 connected = true;
617 break;
618 case RADEON_HPD_2:
619 if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
620 connected = true;
621 break;
622 case RADEON_HPD_3:
623 if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
624 connected = true;
625 break;
626 default:
627 break;
628 }
629 }
630 return connected;
631}
632
633void r600_hpd_set_polarity(struct radeon_device *rdev,
429770b3 634 enum radeon_hpd_id hpd)
e0df1ac5
AD		 635{
636 u32 tmp;
637 bool connected = r600_hpd_sense(rdev, hpd);
638
639 if (ASIC_IS_DCE3(rdev)) {
640 switch (hpd) {
641 case RADEON_HPD_1:
642 tmp = RREG32(DC_HPD1_INT_CONTROL);
643 if (connected)
644 tmp &= ~DC_HPDx_INT_POLARITY;
645 else
646 tmp |= DC_HPDx_INT_POLARITY;
647 WREG32(DC_HPD1_INT_CONTROL, tmp);
648 break;
649 case RADEON_HPD_2:
650 tmp = RREG32(DC_HPD2_INT_CONTROL);
651 if (connected)
652 tmp &= ~DC_HPDx_INT_POLARITY;
653 else
654 tmp |= DC_HPDx_INT_POLARITY;
655 WREG32(DC_HPD2_INT_CONTROL, tmp);
656 break;
657 case RADEON_HPD_3:
658 tmp = RREG32(DC_HPD3_INT_CONTROL);
659 if (connected)
660 tmp &= ~DC_HPDx_INT_POLARITY;
661 else
662 tmp |= DC_HPDx_INT_POLARITY;
663 WREG32(DC_HPD3_INT_CONTROL, tmp);
664 break;
665 case RADEON_HPD_4:
666 tmp = RREG32(DC_HPD4_INT_CONTROL);
667 if (connected)
668 tmp &= ~DC_HPDx_INT_POLARITY;
669 else
670 tmp |= DC_HPDx_INT_POLARITY;
671 WREG32(DC_HPD4_INT_CONTROL, tmp);
672 break;
673 case RADEON_HPD_5:
674 tmp = RREG32(DC_HPD5_INT_CONTROL);
675 if (connected)
676 tmp &= ~DC_HPDx_INT_POLARITY;
677 else
678 tmp |= DC_HPDx_INT_POLARITY;
679 WREG32(DC_HPD5_INT_CONTROL, tmp);
680 break;
681 /* DCE 3.2 */
682 case RADEON_HPD_6:
683 tmp = RREG32(DC_HPD6_INT_CONTROL);
684 if (connected)
685 tmp &= ~DC_HPDx_INT_POLARITY;
686 else
687 tmp |= DC_HPDx_INT_POLARITY;
688 WREG32(DC_HPD6_INT_CONTROL, tmp);
689 break;
690 default:
691 break;
692 }
693 } else {
694 switch (hpd) {
695 case RADEON_HPD_1:
696 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
697 if (connected)
698 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
699 else
700 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
701 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
702 break;
703 case RADEON_HPD_2:
704 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
705 if (connected)
706 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
707 else
708 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
709 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
710 break;
711 case RADEON_HPD_3:
712 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
713 if (connected)
714 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
715 else
716 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
717 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
718 break;
719 default:
720 break;
721 }
722 }
723}
724
725void r600_hpd_init(struct radeon_device *rdev)
726{
727 struct drm_device *dev = rdev->ddev;
728 struct drm_connector *connector;
fb98257a 729 unsigned enable = 0;
e0df1ac5 730
64912e99
AD		 731 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
732 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
733
455c89b9
JG		 734 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
735 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
736 /* don't try to enable hpd on eDP or LVDS avoid breaking the
737 * aux dp channel on imac and help (but not completely fix)
738 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
739 */
740 continue;
741 }
64912e99
AD		 742 if (ASIC_IS_DCE3(rdev)) {
743 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
744 if (ASIC_IS_DCE32(rdev))
745 tmp |= DC_HPDx_EN;
e0df1ac5 746
e0df1ac5
AD		 747 switch (radeon_connector->hpd.hpd) {
748 case RADEON_HPD_1:
749 WREG32(DC_HPD1_CONTROL, tmp);
e0df1ac5
AD		 750 break;
751 case RADEON_HPD_2:
752 WREG32(DC_HPD2_CONTROL, tmp);
e0df1ac5
AD		 753 break;
754 case RADEON_HPD_3:
755 WREG32(DC_HPD3_CONTROL, tmp);
e0df1ac5
AD		 756 break;
757 case RADEON_HPD_4:
758 WREG32(DC_HPD4_CONTROL, tmp);
e0df1ac5
AD		 759 break;
760 /* DCE 3.2 */
761 case RADEON_HPD_5:
762 WREG32(DC_HPD5_CONTROL, tmp);
e0df1ac5
AD		 763 break;
764 case RADEON_HPD_6:
765 WREG32(DC_HPD6_CONTROL, tmp);
e0df1ac5
AD		 766 break;
767 default:
768 break;
769 }
64912e99 770 } else {
e0df1ac5
AD		 771 switch (radeon_connector->hpd.hpd) {
772 case RADEON_HPD_1:
773 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
e0df1ac5
AD		 774 break;
775 case RADEON_HPD_2:
776 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
e0df1ac5
AD		 777 break;
778 case RADEON_HPD_3:
779 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
e0df1ac5
AD		 780 break;
781 default:
782 break;
783 }
784 }
fb98257a 785 enable |= 1 << radeon_connector->hpd.hpd;
64912e99 786 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
e0df1ac5 787 }
fb98257a 788 radeon_irq_kms_enable_hpd(rdev, enable);
e0df1ac5
AD		 789}
790
791void r600_hpd_fini(struct radeon_device *rdev)
792{
793 struct drm_device *dev = rdev->ddev;
794 struct drm_connector *connector;
fb98257a 795 unsigned disable = 0;
e0df1ac5 796
fb98257a
CK		 797 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
798 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
799 if (ASIC_IS_DCE3(rdev)) {
e0df1ac5
AD		 800 switch (radeon_connector->hpd.hpd) {
801 case RADEON_HPD_1:
802 WREG32(DC_HPD1_CONTROL, 0);
e0df1ac5
AD		 803 break;
804 case RADEON_HPD_2:
805 WREG32(DC_HPD2_CONTROL, 0);
e0df1ac5
AD		 806 break;
807 case RADEON_HPD_3:
808 WREG32(DC_HPD3_CONTROL, 0);
e0df1ac5
AD		 809 break;
810 case RADEON_HPD_4:
811 WREG32(DC_HPD4_CONTROL, 0);
e0df1ac5
AD		 812 break;
813 /* DCE 3.2 */
814 case RADEON_HPD_5:
815 WREG32(DC_HPD5_CONTROL, 0);
e0df1ac5
AD		 816 break;
817 case RADEON_HPD_6:
818 WREG32(DC_HPD6_CONTROL, 0);
e0df1ac5
AD		 819 break;
820 default:
821 break;
822 }
fb98257a 823 } else {
e0df1ac5
AD		 824 switch (radeon_connector->hpd.hpd) {
825 case RADEON_HPD_1:
826 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
e0df1ac5
AD		 827 break;
828 case RADEON_HPD_2:
829 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
e0df1ac5
AD		 830 break;
831 case RADEON_HPD_3:
832 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
e0df1ac5
AD		 833 break;
834 default:
835 break;
836 }
837 }
fb98257a 838 disable |= 1 << radeon_connector->hpd.hpd;
e0df1ac5 839 }
fb98257a 840 radeon_irq_kms_disable_hpd(rdev, disable);
e0df1ac5
AD		 841}
842
771fe6b9 843/*
3ce0a23d 844 * R600 PCIE GART
771fe6b9 845 */
3ce0a23d
JG		 846void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
847{
848 unsigned i;
849 u32 tmp;
850
2e98f10a 851 /* flush hdp cache so updates hit vram */
f3886f85
AD		 852 if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
853 !(rdev->flags & RADEON_IS_AGP)) {
c9a1be96 854 void __iomem *ptr = (void *)rdev->gart.ptr;
812d0469
AD		 855 u32 tmp;
856
857 /* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
858 * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
f3886f85
AD		 859 * This seems to cause problems on some AGP cards. Just use the old
860 * method for them.
812d0469
AD		 861 */
862 WREG32(HDP_DEBUG1, 0);
863 tmp = readl((void __iomem *)ptr);
864 } else
865 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2e98f10a 866
3ce0a23d
JG		 867 WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
868 WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
869 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
870 for (i = 0; i < rdev->usec_timeout; i++) {
871 /* read MC_STATUS */
872 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
873 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
874 if (tmp == 2) {
875 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
876 return;
877 }
878 if (tmp) {
879 return;
880 }
881 udelay(1);
882 }
883}
884
4aac0473 885int r600_pcie_gart_init(struct radeon_device *rdev)
3ce0a23d 886{
4aac0473 887 int r;
3ce0a23d 888
c9a1be96 889 if (rdev->gart.robj) {
fce7d61b 890 WARN(1, "R600 PCIE GART already initialized\n");
4aac0473
JG		 891 return 0;
892 }
3ce0a23d
JG		 893 /* Initialize common gart structure */
894 r = radeon_gart_init(rdev);
4aac0473 895 if (r)
3ce0a23d 896 return r;
3ce0a23d 897 rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
4aac0473
JG		 898 return radeon_gart_table_vram_alloc(rdev);
899}
900
1109ca09 901static int r600_pcie_gart_enable(struct radeon_device *rdev)
4aac0473
JG		 902{
903 u32 tmp;
904 int r, i;
905
c9a1be96 906 if (rdev->gart.robj == NULL) {
4aac0473
JG		 907 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
908 return -EINVAL;
771fe6b9 909 }
4aac0473
JG		 910 r = radeon_gart_table_vram_pin(rdev);
911 if (r)
912 return r;
82568565 913 radeon_gart_restore(rdev);
bc1a631e 914
3ce0a23d
JG		 915 /* Setup L2 cache */
916 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
917 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
918 EFFECTIVE_L2_QUEUE_SIZE(7));
919 WREG32(VM_L2_CNTL2, 0);
920 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
921 /* Setup TLB control */
922 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
923 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
924 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
925 ENABLE_WAIT_L2_QUERY;
926 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
927 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
928 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
929 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
930 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
931 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
932 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
933 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
934 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
935 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
936 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
937 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
938 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
939 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
940 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1a029b76 941 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3ce0a23d
JG		 942 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
943 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
944 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
945 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
946 (u32)(rdev->dummy_page.addr >> 12));
947 for (i = 1; i < 7; i++)
948 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
771fe6b9 949
3ce0a23d 950 r600_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 951 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
952 (unsigned)(rdev->mc.gtt_size >> 20),
953 (unsigned long long)rdev->gart.table_addr);
3ce0a23d 954 rdev->gart.ready = true;
771fe6b9
JG		 955 return 0;
956}
957
1109ca09 958static void r600_pcie_gart_disable(struct radeon_device *rdev)
771fe6b9 959{
3ce0a23d 960 u32 tmp;
c9a1be96 961 int i;
771fe6b9 962
3ce0a23d
JG		 963 /* Disable all tables */
964 for (i = 0; i < 7; i++)
965 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
771fe6b9 966
3ce0a23d
JG		 967 /* Disable L2 cache */
968 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
969 EFFECTIVE_L2_QUEUE_SIZE(7));
970 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
971 /* Setup L1 TLB control */
972 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
973 ENABLE_WAIT_L2_QUERY;
974 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
975 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
976 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
977 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
978 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
979 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
980 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
981 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
982 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
983 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
984 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
985 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
986 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
987 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
c9a1be96 988 radeon_gart_table_vram_unpin(rdev);
4aac0473
JG		 989}
990
1109ca09 991static void r600_pcie_gart_fini(struct radeon_device *rdev)
4aac0473 992{
f9274562 993 radeon_gart_fini(rdev);
4aac0473
JG		 994 r600_pcie_gart_disable(rdev);
995 radeon_gart_table_vram_free(rdev);
771fe6b9
JG		 996}
997
1109ca09 998static void r600_agp_enable(struct radeon_device *rdev)
1a029b76
JG		 999{
1000 u32 tmp;
1001 int i;
1002
1003 /* Setup L2 cache */
1004 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1005 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1006 EFFECTIVE_L2_QUEUE_SIZE(7));
1007 WREG32(VM_L2_CNTL2, 0);
1008 WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
1009 /* Setup TLB control */
1010 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1011 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1012 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1013 ENABLE_WAIT_L2_QUERY;
1014 WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1015 WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1016 WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
1017 WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1018 WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1019 WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1020 WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1021 WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1022 WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1023 WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1024 WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1025 WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1026 WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1027 WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1028 for (i = 0; i < 7; i++)
1029 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1030}
1031
771fe6b9
JG		 1032int r600_mc_wait_for_idle(struct radeon_device *rdev)
1033{
3ce0a23d
JG		 1034 unsigned i;
1035 u32 tmp;
1036
1037 for (i = 0; i < rdev->usec_timeout; i++) {
1038 /* read MC_STATUS */
1039 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
1040 if (!tmp)
1041 return 0;
1042 udelay(1);
1043 }
1044 return -1;
771fe6b9
JG		 1045}
1046
65337e60
SL		 1047uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg)
1048{
1049 uint32_t r;
1050
1051 WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg));
1052 r = RREG32(R_0028FC_MC_DATA);
1053 WREG32(R_0028F8_MC_INDEX, ~C_0028F8_MC_IND_ADDR);
1054 return r;
1055}
1056
1057void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
1058{
1059 WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg) |
1060 S_0028F8_MC_IND_WR_EN(1));
1061 WREG32(R_0028FC_MC_DATA, v);
1062 WREG32(R_0028F8_MC_INDEX, 0x7F);
1063}
1064
a3c1945a 1065static void r600_mc_program(struct radeon_device *rdev)
771fe6b9 1066{
a3c1945a 1067 struct rv515_mc_save save;
3ce0a23d
JG		 1068 u32 tmp;
1069 int i, j;
771fe6b9 1070
3ce0a23d
JG		 1071 /* Initialize HDP */
1072 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1073 WREG32((0x2c14 + j), 0x00000000);
1074 WREG32((0x2c18 + j), 0x00000000);
1075 WREG32((0x2c1c + j), 0x00000000);
1076 WREG32((0x2c20 + j), 0x00000000);
1077 WREG32((0x2c24 + j), 0x00000000);
1078 }
1079 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
771fe6b9 1080
a3c1945a 1081 rv515_mc_stop(rdev, &save);
3ce0a23d 1082 if (r600_mc_wait_for_idle(rdev)) {
a3c1945a 1083 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3ce0a23d 1084 }
a3c1945a 1085 /* Lockout access through VGA aperture (doesn't exist before R600) */
3ce0a23d 1086 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3ce0a23d 1087 /* Update configuration */
1a029b76
JG		 1088 if (rdev->flags & RADEON_IS_AGP) {
1089 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1090 /* VRAM before AGP */
1091 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1092 rdev->mc.vram_start >> 12);
1093 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1094 rdev->mc.gtt_end >> 12);
1095 } else {
1096 /* VRAM after AGP */
1097 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1098 rdev->mc.gtt_start >> 12);
1099 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1100 rdev->mc.vram_end >> 12);
1101 }
1102 } else {
1103 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
1104 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
1105 }
16cdf04d 1106 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1a029b76 1107 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3ce0a23d
JG		 1108 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1109 WREG32(MC_VM_FB_LOCATION, tmp);
1110 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1111 WREG32(HDP_NONSURFACE_INFO, (2 << 7));
46fcd2b3 1112 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3ce0a23d 1113 if (rdev->flags & RADEON_IS_AGP) {
1a029b76
JG		 1114 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
1115 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
3ce0a23d
JG		 1116 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1117 } else {
1118 WREG32(MC_VM_AGP_BASE, 0);
1119 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1120 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1121 }
3ce0a23d 1122 if (r600_mc_wait_for_idle(rdev)) {
a3c1945a 1123 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3ce0a23d 1124 }
a3c1945a 1125 rv515_mc_resume(rdev, &save);
698443d9
DA		 1126 /* we need to own VRAM, so turn off the VGA renderer here
1127 * to stop it overwriting our objects */
d39c3b89 1128 rv515_vga_render_disable(rdev);
3ce0a23d
JG		 1129}
1130
d594e46a
JG		 1131/**
1132 * r600_vram_gtt_location - try to find VRAM & GTT location
1133 * @rdev: radeon device structure holding all necessary informations
1134 * @mc: memory controller structure holding memory informations
1135 *
1136 * Function will place try to place VRAM at same place as in CPU (PCI)
1137 * address space as some GPU seems to have issue when we reprogram at
1138 * different address space.
1139 *
1140 * If there is not enough space to fit the unvisible VRAM after the
1141 * aperture then we limit the VRAM size to the aperture.
1142 *
1143 * If we are using AGP then place VRAM adjacent to AGP aperture are we need
1144 * them to be in one from GPU point of view so that we can program GPU to
1145 * catch access outside them (weird GPU policy see ??).
1146 *
1147 * This function will never fails, worst case are limiting VRAM or GTT.
1148 *
1149 * Note: GTT start, end, size should be initialized before calling this
1150 * function on AGP platform.
1151 */
0ef0c1f7 1152static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
d594e46a
JG		 1153{
1154 u64 size_bf, size_af;
1155
1156 if (mc->mc_vram_size > 0xE0000000) {
1157 /* leave room for at least 512M GTT */
1158 dev_warn(rdev->dev, "limiting VRAM\n");
1159 mc->real_vram_size = 0xE0000000;
1160 mc->mc_vram_size = 0xE0000000;
1161 }
1162 if (rdev->flags & RADEON_IS_AGP) {
1163 size_bf = mc->gtt_start;
9ed8b1f9 1164 size_af = mc->mc_mask - mc->gtt_end;
d594e46a
JG		 1165 if (size_bf > size_af) {
1166 if (mc->mc_vram_size > size_bf) {
1167 dev_warn(rdev->dev, "limiting VRAM\n");
1168 mc->real_vram_size = size_bf;
1169 mc->mc_vram_size = size_bf;
1170 }
1171 mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1172 } else {
1173 if (mc->mc_vram_size > size_af) {
1174 dev_warn(rdev->dev, "limiting VRAM\n");
1175 mc->real_vram_size = size_af;
1176 mc->mc_vram_size = size_af;
1177 }
dfc6ae5b 1178 mc->vram_start = mc->gtt_end + 1;
d594e46a
JG		 1179 }
1180 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1181 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1182 mc->mc_vram_size >> 20, mc->vram_start,
1183 mc->vram_end, mc->real_vram_size >> 20);
1184 } else {
1185 u64 base = 0;
8961d52d
AD		 1186 if (rdev->flags & RADEON_IS_IGP) {
1187 base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
1188 base <<= 24;
1189 }
d594e46a 1190 radeon_vram_location(rdev, &rdev->mc, base);
8d369bb1 1191 rdev->mc.gtt_base_align = 0;
d594e46a
JG		 1192 radeon_gtt_location(rdev, mc);
1193 }
1194}
1195
1109ca09 1196static int r600_mc_init(struct radeon_device *rdev)
771fe6b9 1197{
3ce0a23d 1198 u32 tmp;
5885b7a9 1199 int chansize, numchan;
65337e60
SL		 1200 uint32_t h_addr, l_addr;
1201 unsigned long long k8_addr;
771fe6b9 1202
3ce0a23d 1203 /* Get VRAM informations */
771fe6b9 1204 rdev->mc.vram_is_ddr = true;
3ce0a23d
JG		 1205 tmp = RREG32(RAMCFG);
1206 if (tmp & CHANSIZE_OVERRIDE) {
771fe6b9 1207 chansize = 16;
3ce0a23d 1208 } else if (tmp & CHANSIZE_MASK) {
771fe6b9
JG		 1209 chansize = 64;
1210 } else {
1211 chansize = 32;
1212 }
5885b7a9
AD		 1213 tmp = RREG32(CHMAP);
1214 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1215 case 0:
1216 default:
1217 numchan = 1;
1218 break;
1219 case 1:
1220 numchan = 2;
1221 break;
1222 case 2:
1223 numchan = 4;
1224 break;
1225 case 3:
1226 numchan = 8;
1227 break;
771fe6b9 1228 }
5885b7a9 1229 rdev->mc.vram_width = numchan * chansize;
3ce0a23d 1230 /* Could aper size report 0 ? */
01d73a69
JC		 1231 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1232 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3ce0a23d
JG		 1233 /* Setup GPU memory space */
1234 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1235 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
51e5fcd3 1236 rdev->mc.visible_vram_size = rdev->mc.aper_size;
d594e46a 1237 r600_vram_gtt_location(rdev, &rdev->mc);
f47299c5 1238
f892034a
AD		 1239 if (rdev->flags & RADEON_IS_IGP) {
1240 rs690_pm_info(rdev);
06b6476d 1241 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
65337e60
SL		 1242
1243 if (rdev->family == CHIP_RS780 || rdev->family == CHIP_RS880) {
1244 /* Use K8 direct mapping for fast fb access. */
1245 rdev->fastfb_working = false;
1246 h_addr = G_000012_K8_ADDR_EXT(RREG32_MC(R_000012_MC_MISC_UMA_CNTL));
1247 l_addr = RREG32_MC(R_000011_K8_FB_LOCATION);
1248 k8_addr = ((unsigned long long)h_addr) << 32 | l_addr;
1249#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
1250 if (k8_addr + rdev->mc.visible_vram_size < 0x100000000ULL)
1251#endif
1252 {
1253 /* FastFB shall be used with UMA memory. Here it is simply disabled when sideport
1254 * memory is present.
1255 */
1256 if (rdev->mc.igp_sideport_enabled == false && radeon_fastfb == 1) {
1257 DRM_INFO("Direct mapping: aper base at 0x%llx, replaced by direct mapping base 0x%llx.\n",
1258 (unsigned long long)rdev->mc.aper_base, k8_addr);
1259 rdev->mc.aper_base = (resource_size_t)k8_addr;
1260 rdev->fastfb_working = true;
1261 }
1262 }
1263 }
f892034a 1264 }
65337e60 1265
f47299c5 1266 radeon_update_bandwidth_info(rdev);
3ce0a23d 1267 return 0;
771fe6b9
JG		 1268}
1269
16cdf04d
AD		 1270int r600_vram_scratch_init(struct radeon_device *rdev)
1271{
1272 int r;
1273
1274 if (rdev->vram_scratch.robj == NULL) {
1275 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
1276 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
40f5cf99 1277 NULL, &rdev->vram_scratch.robj);
16cdf04d
AD		 1278 if (r) {
1279 return r;
1280 }
1281 }
1282
1283 r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1284 if (unlikely(r != 0))
1285 return r;
1286 r = radeon_bo_pin(rdev->vram_scratch.robj,
1287 RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
1288 if (r) {
1289 radeon_bo_unreserve(rdev->vram_scratch.robj);
1290 return r;
1291 }
1292 r = radeon_bo_kmap(rdev->vram_scratch.robj,
1293 (void **)&rdev->vram_scratch.ptr);
1294 if (r)
1295 radeon_bo_unpin(rdev->vram_scratch.robj);
1296 radeon_bo_unreserve(rdev->vram_scratch.robj);
1297
1298 return r;
1299}
1300
1301void r600_vram_scratch_fini(struct radeon_device *rdev)
1302{
1303 int r;
1304
1305 if (rdev->vram_scratch.robj == NULL) {
1306 return;
1307 }
1308 r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1309 if (likely(r == 0)) {
1310 radeon_bo_kunmap(rdev->vram_scratch.robj);
1311 radeon_bo_unpin(rdev->vram_scratch.robj);
1312 radeon_bo_unreserve(rdev->vram_scratch.robj);
1313 }
1314 radeon_bo_unref(&rdev->vram_scratch.robj);
1315}
1316
410a3418
AD		 1317void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)
1318{
1319 u32 tmp = RREG32(R600_BIOS_3_SCRATCH);
1320
1321 if (hung)
1322 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1323 else
1324 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1325
1326 WREG32(R600_BIOS_3_SCRATCH, tmp);
1327}
1328
d3cb781e 1329static void r600_print_gpu_status_regs(struct radeon_device *rdev)
771fe6b9 1330{
64c56e8c 1331 dev_info(rdev->dev, " R_008010_GRBM_STATUS = 0x%08X\n",
d3cb781e 1332 RREG32(R_008010_GRBM_STATUS));
64c56e8c 1333 dev_info(rdev->dev, " R_008014_GRBM_STATUS2 = 0x%08X\n",
d3cb781e 1334 RREG32(R_008014_GRBM_STATUS2));
64c56e8c 1335 dev_info(rdev->dev, " R_000E50_SRBM_STATUS = 0x%08X\n",
d3cb781e 1336 RREG32(R_000E50_SRBM_STATUS));
440a7cd8 1337 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
d3cb781e 1338 RREG32(CP_STALLED_STAT1));
440a7cd8 1339 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
d3cb781e 1340 RREG32(CP_STALLED_STAT2));
440a7cd8 1341 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
d3cb781e 1342 RREG32(CP_BUSY_STAT));
440a7cd8 1343 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
d3cb781e 1344 RREG32(CP_STAT));
71e3d157
AD		 1345 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
1346 RREG32(DMA_STATUS_REG));
1347}
1348
f13f7731 1349static bool r600_is_display_hung(struct radeon_device *rdev)
71e3d157 1350{
f13f7731
AD		 1351 u32 crtc_hung = 0;
1352 u32 crtc_status[2];
1353 u32 i, j, tmp;
1354
1355 for (i = 0; i < rdev->num_crtc; i++) {
1356 if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[i]) & AVIVO_CRTC_EN) {
1357 crtc_status[i] = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1358 crtc_hung |= (1 << i);
1359 }
1360 }
1361
1362 for (j = 0; j < 10; j++) {
1363 for (i = 0; i < rdev->num_crtc; i++) {
1364 if (crtc_hung & (1 << i)) {
1365 tmp = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1366 if (tmp != crtc_status[i])
1367 crtc_hung &= ~(1 << i);
1368 }
1369 }
1370 if (crtc_hung == 0)
1371 return false;
1372 udelay(100);
1373 }
1374
1375 return true;
1376}
1377
1378static u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)
1379{
1380 u32 reset_mask = 0;
d3cb781e 1381 u32 tmp;
71e3d157 1382
f13f7731
AD		 1383 /* GRBM_STATUS */
1384 tmp = RREG32(R_008010_GRBM_STATUS);
1385 if (rdev->family >= CHIP_RV770) {
1386 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1387 G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1388 G_008010_TA_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1389 G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1390 G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1391 reset_mask |= RADEON_RESET_GFX;
1392 } else {
1393 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1394 G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1395 G_008010_TA03_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1396 G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1397 G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1398 reset_mask |= RADEON_RESET_GFX;
1399 }
1400
1401 if (G_008010_CF_RQ_PENDING(tmp) | G_008010_PF_RQ_PENDING(tmp) |
1402 G_008010_CP_BUSY(tmp) | G_008010_CP_COHERENCY_BUSY(tmp))
1403 reset_mask |= RADEON_RESET_CP;
1404
1405 if (G_008010_GRBM_EE_BUSY(tmp))
1406 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1407
1408 /* DMA_STATUS_REG */
1409 tmp = RREG32(DMA_STATUS_REG);
1410 if (!(tmp & DMA_IDLE))
1411 reset_mask |= RADEON_RESET_DMA;
1412
1413 /* SRBM_STATUS */
1414 tmp = RREG32(R_000E50_SRBM_STATUS);
1415 if (G_000E50_RLC_RQ_PENDING(tmp) | G_000E50_RLC_BUSY(tmp))
1416 reset_mask |= RADEON_RESET_RLC;
1417
1418 if (G_000E50_IH_BUSY(tmp))
1419 reset_mask |= RADEON_RESET_IH;
1420
1421 if (G_000E50_SEM_BUSY(tmp))
1422 reset_mask |= RADEON_RESET_SEM;
19fc42ed 1423
f13f7731
AD		 1424 if (G_000E50_GRBM_RQ_PENDING(tmp))
1425 reset_mask |= RADEON_RESET_GRBM;
1426
1427 if (G_000E50_VMC_BUSY(tmp))
1428 reset_mask |= RADEON_RESET_VMC;
1429
1430 if (G_000E50_MCB_BUSY(tmp) | G_000E50_MCDZ_BUSY(tmp) |
1431 G_000E50_MCDY_BUSY(tmp) | G_000E50_MCDX_BUSY(tmp) |
1432 G_000E50_MCDW_BUSY(tmp))
1433 reset_mask |= RADEON_RESET_MC;
1434
1435 if (r600_is_display_hung(rdev))
1436 reset_mask |= RADEON_RESET_DISPLAY;
1437
d808fc88
AD		 1438 /* Skip MC reset as it's mostly likely not hung, just busy */
1439 if (reset_mask & RADEON_RESET_MC) {
1440 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1441 reset_mask &= ~RADEON_RESET_MC;
1442 }
1443
f13f7731
AD		 1444 return reset_mask;
1445}
1446
1447static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1448{
1449 struct rv515_mc_save save;
1450 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1451 u32 tmp;
19fc42ed 1452
71e3d157 1453 if (reset_mask == 0)
f13f7731 1454 return;
71e3d157
AD		 1455
1456 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1457
d3cb781e
AD		 1458 r600_print_gpu_status_regs(rdev);
1459
d3cb781e
AD		 1460 /* Disable CP parsing/prefetching */
1461 if (rdev->family >= CHIP_RV770)
1462 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) | S_0086D8_CP_PFP_HALT(1));
1463 else
1464 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1465
1466 /* disable the RLC */
1467 WREG32(RLC_CNTL, 0);
1468
1469 if (reset_mask & RADEON_RESET_DMA) {
1470 /* Disable DMA */
1471 tmp = RREG32(DMA_RB_CNTL);
1472 tmp &= ~DMA_RB_ENABLE;
1473 WREG32(DMA_RB_CNTL, tmp);
1474 }
1475
1476 mdelay(50);
1477
ca57802e
AD		 1478 rv515_mc_stop(rdev, &save);
1479 if (r600_mc_wait_for_idle(rdev)) {
1480 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1481 }
1482
d3cb781e
AD		 1483 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1484 if (rdev->family >= CHIP_RV770)
1485 grbm_soft_reset |= S_008020_SOFT_RESET_DB(1) |
1486 S_008020_SOFT_RESET_CB(1) |
1487 S_008020_SOFT_RESET_PA(1) |
1488 S_008020_SOFT_RESET_SC(1) |
1489 S_008020_SOFT_RESET_SPI(1) |
1490 S_008020_SOFT_RESET_SX(1) |
1491 S_008020_SOFT_RESET_SH(1) |
1492 S_008020_SOFT_RESET_TC(1) |
1493 S_008020_SOFT_RESET_TA(1) |
1494 S_008020_SOFT_RESET_VC(1) |
1495 S_008020_SOFT_RESET_VGT(1);
1496 else
1497 grbm_soft_reset |= S_008020_SOFT_RESET_CR(1) |
1498 S_008020_SOFT_RESET_DB(1) |
1499 S_008020_SOFT_RESET_CB(1) |
1500 S_008020_SOFT_RESET_PA(1) |
1501 S_008020_SOFT_RESET_SC(1) |
1502 S_008020_SOFT_RESET_SMX(1) |
1503 S_008020_SOFT_RESET_SPI(1) |
1504 S_008020_SOFT_RESET_SX(1) |
1505 S_008020_SOFT_RESET_SH(1) |
1506 S_008020_SOFT_RESET_TC(1) |
1507 S_008020_SOFT_RESET_TA(1) |
1508 S_008020_SOFT_RESET_VC(1) |
1509 S_008020_SOFT_RESET_VGT(1);
1510 }
1511
1512 if (reset_mask & RADEON_RESET_CP) {
1513 grbm_soft_reset |= S_008020_SOFT_RESET_CP(1) |
1514 S_008020_SOFT_RESET_VGT(1);
1515
1516 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1517 }
1518
1519 if (reset_mask & RADEON_RESET_DMA) {
1520 if (rdev->family >= CHIP_RV770)
1521 srbm_soft_reset |= RV770_SOFT_RESET_DMA;
1522 else
1523 srbm_soft_reset |= SOFT_RESET_DMA;
1524 }
1525
f13f7731
AD		 1526 if (reset_mask & RADEON_RESET_RLC)
1527 srbm_soft_reset |= S_000E60_SOFT_RESET_RLC(1);
1528
1529 if (reset_mask & RADEON_RESET_SEM)
1530 srbm_soft_reset |= S_000E60_SOFT_RESET_SEM(1);
1531
1532 if (reset_mask & RADEON_RESET_IH)
1533 srbm_soft_reset |= S_000E60_SOFT_RESET_IH(1);
1534
1535 if (reset_mask & RADEON_RESET_GRBM)
1536 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1537
24178ec4
AD		 1538 if (!(rdev->flags & RADEON_IS_IGP)) {
1539 if (reset_mask & RADEON_RESET_MC)
1540 srbm_soft_reset |= S_000E60_SOFT_RESET_MC(1);
1541 }
f13f7731
AD		 1542
1543 if (reset_mask & RADEON_RESET_VMC)
1544 srbm_soft_reset |= S_000E60_SOFT_RESET_VMC(1);
1545
d3cb781e
AD		 1546 if (grbm_soft_reset) {
1547 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1548 tmp |= grbm_soft_reset;
1549 dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1550 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1551 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1552
1553 udelay(50);
1554
1555 tmp &= ~grbm_soft_reset;
1556 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1557 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1558 }
1559
1560 if (srbm_soft_reset) {
1561 tmp = RREG32(SRBM_SOFT_RESET);
1562 tmp |= srbm_soft_reset;
1563 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1564 WREG32(SRBM_SOFT_RESET, tmp);
1565 tmp = RREG32(SRBM_SOFT_RESET);
1566
1567 udelay(50);
71e3d157 1568
d3cb781e
AD		 1569 tmp &= ~srbm_soft_reset;
1570 WREG32(SRBM_SOFT_RESET, tmp);
1571 tmp = RREG32(SRBM_SOFT_RESET);
1572 }
71e3d157
AD		 1573
1574 /* Wait a little for things to settle down */
1575 mdelay(1);
1576
a3c1945a 1577 rv515_mc_resume(rdev, &save);
d3cb781e 1578 udelay(50);
410a3418 1579
d3cb781e 1580 r600_print_gpu_status_regs(rdev);
d3cb781e
AD		 1581}
1582
1583int r600_asic_reset(struct radeon_device *rdev)
1584{
f13f7731
AD		 1585 u32 reset_mask;
1586
1587 reset_mask = r600_gpu_check_soft_reset(rdev);
1588
1589 if (reset_mask)
1590 r600_set_bios_scratch_engine_hung(rdev, true);
1591
1592 r600_gpu_soft_reset(rdev, reset_mask);
1593
1594 reset_mask = r600_gpu_check_soft_reset(rdev);
1595
1596 if (!reset_mask)
1597 r600_set_bios_scratch_engine_hung(rdev, false);
1598
1599 return 0;
3ce0a23d
JG		 1600}
1601
123bc183
AD		 1602/**
1603 * r600_gfx_is_lockup - Check if the GFX engine is locked up
1604 *
1605 * @rdev: radeon_device pointer
1606 * @ring: radeon_ring structure holding ring information
1607 *
1608 * Check if the GFX engine is locked up.
1609 * Returns true if the engine appears to be locked up, false if not.
1610 */
1611bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
225758d8 1612{
123bc183
AD		 1613 u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1614
1615 if (!(reset_mask & (RADEON_RESET_GFX |
1616 RADEON_RESET_COMPUTE |
1617 RADEON_RESET_CP))) {
069211e5 1618 radeon_ring_lockup_update(ring);
225758d8
JG		 1619 return false;
1620 }
1621 /* force CP activities */
7b9ef16b 1622 radeon_ring_force_activity(rdev, ring);
069211e5 1623 return radeon_ring_test_lockup(rdev, ring);
225758d8
JG		 1624}
1625
4d75658b
AD		 1626/**
1627 * r600_dma_is_lockup - Check if the DMA engine is locked up
1628 *
1629 * @rdev: radeon_device pointer
1630 * @ring: radeon_ring structure holding ring information
1631 *
123bc183 1632 * Check if the async DMA engine is locked up.
4d75658b
AD		 1633 * Returns true if the engine appears to be locked up, false if not.
1634 */
1635bool r600_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1636{
123bc183 1637 u32 reset_mask = r600_gpu_check_soft_reset(rdev);
4d75658b 1638
123bc183 1639 if (!(reset_mask & RADEON_RESET_DMA)) {
4d75658b
AD		 1640 radeon_ring_lockup_update(ring);
1641 return false;
1642 }
1643 /* force ring activities */
1644 radeon_ring_force_activity(rdev, ring);
1645 return radeon_ring_test_lockup(rdev, ring);
1646}
1647
416a2bd2
AD		 1648u32 r6xx_remap_render_backend(struct radeon_device *rdev,
1649 u32 tiling_pipe_num,
1650 u32 max_rb_num,
1651 u32 total_max_rb_num,
1652 u32 disabled_rb_mask)
3ce0a23d 1653{
416a2bd2 1654 u32 rendering_pipe_num, rb_num_width, req_rb_num;
f689e3ac 1655 u32 pipe_rb_ratio, pipe_rb_remain, tmp;
416a2bd2
AD		 1656 u32 data = 0, mask = 1 << (max_rb_num - 1);
1657 unsigned i, j;
3ce0a23d 1658
416a2bd2 1659 /* mask out the RBs that don't exist on that asic */
f689e3ac
MT		 1660 tmp = disabled_rb_mask | ((0xff << max_rb_num) & 0xff);
1661 /* make sure at least one RB is available */
1662 if ((tmp & 0xff) != 0xff)
1663 disabled_rb_mask = tmp;
3ce0a23d 1664
416a2bd2
AD		 1665 rendering_pipe_num = 1 << tiling_pipe_num;
1666 req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
1667 BUG_ON(rendering_pipe_num < req_rb_num);
3ce0a23d 1668
416a2bd2
AD		 1669 pipe_rb_ratio = rendering_pipe_num / req_rb_num;
1670 pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
3ce0a23d 1671
416a2bd2
AD		 1672 if (rdev->family <= CHIP_RV740) {
1673 /* r6xx/r7xx */
1674 rb_num_width = 2;
1675 } else {
1676 /* eg+ */
1677 rb_num_width = 4;
1678 }
3ce0a23d 1679
416a2bd2
AD		 1680 for (i = 0; i < max_rb_num; i++) {
1681 if (!(mask & disabled_rb_mask)) {
1682 for (j = 0; j < pipe_rb_ratio; j++) {
1683 data <<= rb_num_width;
1684 data |= max_rb_num - i - 1;
1685 }
1686 if (pipe_rb_remain) {
1687 data <<= rb_num_width;
1688 data |= max_rb_num - i - 1;
1689 pipe_rb_remain--;
1690 }
1691 }
1692 mask >>= 1;
3ce0a23d
JG		 1693 }
1694
416a2bd2 1695 return data;
3ce0a23d
JG		 1696}
1697
1698int r600_count_pipe_bits(uint32_t val)
1699{
ef8cf3a1 1700 return hweight32(val);
771fe6b9
JG		 1701}
1702
1109ca09 1703static void r600_gpu_init(struct radeon_device *rdev)
3ce0a23d
JG		 1704{
1705 u32 tiling_config;
1706 u32 ramcfg;
d03f5d59
AD		 1707 u32 cc_rb_backend_disable;
1708 u32 cc_gc_shader_pipe_config;
3ce0a23d
JG		 1709 u32 tmp;
1710 int i, j;
1711 u32 sq_config;
1712 u32 sq_gpr_resource_mgmt_1 = 0;
1713 u32 sq_gpr_resource_mgmt_2 = 0;
1714 u32 sq_thread_resource_mgmt = 0;
1715 u32 sq_stack_resource_mgmt_1 = 0;
1716 u32 sq_stack_resource_mgmt_2 = 0;
416a2bd2 1717 u32 disabled_rb_mask;
3ce0a23d 1718
416a2bd2 1719 rdev->config.r600.tiling_group_size = 256;
3ce0a23d
JG		 1720 switch (rdev->family) {
1721 case CHIP_R600:
1722 rdev->config.r600.max_pipes = 4;
1723 rdev->config.r600.max_tile_pipes = 8;
1724 rdev->config.r600.max_simds = 4;
1725 rdev->config.r600.max_backends = 4;
1726 rdev->config.r600.max_gprs = 256;
1727 rdev->config.r600.max_threads = 192;
1728 rdev->config.r600.max_stack_entries = 256;
1729 rdev->config.r600.max_hw_contexts = 8;
1730 rdev->config.r600.max_gs_threads = 16;
1731 rdev->config.r600.sx_max_export_size = 128;
1732 rdev->config.r600.sx_max_export_pos_size = 16;
1733 rdev->config.r600.sx_max_export_smx_size = 128;
1734 rdev->config.r600.sq_num_cf_insts = 2;
1735 break;
1736 case CHIP_RV630:
1737 case CHIP_RV635:
1738 rdev->config.r600.max_pipes = 2;
1739 rdev->config.r600.max_tile_pipes = 2;
1740 rdev->config.r600.max_simds = 3;
1741 rdev->config.r600.max_backends = 1;
1742 rdev->config.r600.max_gprs = 128;
1743 rdev->config.r600.max_threads = 192;
1744 rdev->config.r600.max_stack_entries = 128;
1745 rdev->config.r600.max_hw_contexts = 8;
1746 rdev->config.r600.max_gs_threads = 4;
1747 rdev->config.r600.sx_max_export_size = 128;
1748 rdev->config.r600.sx_max_export_pos_size = 16;
1749 rdev->config.r600.sx_max_export_smx_size = 128;
1750 rdev->config.r600.sq_num_cf_insts = 2;
1751 break;
1752 case CHIP_RV610:
1753 case CHIP_RV620:
1754 case CHIP_RS780:
1755 case CHIP_RS880:
1756 rdev->config.r600.max_pipes = 1;
1757 rdev->config.r600.max_tile_pipes = 1;
1758 rdev->config.r600.max_simds = 2;
1759 rdev->config.r600.max_backends = 1;
1760 rdev->config.r600.max_gprs = 128;
1761 rdev->config.r600.max_threads = 192;
1762 rdev->config.r600.max_stack_entries = 128;
1763 rdev->config.r600.max_hw_contexts = 4;
1764 rdev->config.r600.max_gs_threads = 4;
1765 rdev->config.r600.sx_max_export_size = 128;
1766 rdev->config.r600.sx_max_export_pos_size = 16;
1767 rdev->config.r600.sx_max_export_smx_size = 128;
1768 rdev->config.r600.sq_num_cf_insts = 1;
1769 break;
1770 case CHIP_RV670:
1771 rdev->config.r600.max_pipes = 4;
1772 rdev->config.r600.max_tile_pipes = 4;
1773 rdev->config.r600.max_simds = 4;
1774 rdev->config.r600.max_backends = 4;
1775 rdev->config.r600.max_gprs = 192;
1776 rdev->config.r600.max_threads = 192;
1777 rdev->config.r600.max_stack_entries = 256;
1778 rdev->config.r600.max_hw_contexts = 8;
1779 rdev->config.r600.max_gs_threads = 16;
1780 rdev->config.r600.sx_max_export_size = 128;
1781 rdev->config.r600.sx_max_export_pos_size = 16;
1782 rdev->config.r600.sx_max_export_smx_size = 128;
1783 rdev->config.r600.sq_num_cf_insts = 2;
1784 break;
1785 default:
1786 break;
1787 }
1788
1789 /* Initialize HDP */
1790 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1791 WREG32((0x2c14 + j), 0x00000000);
1792 WREG32((0x2c18 + j), 0x00000000);
1793 WREG32((0x2c1c + j), 0x00000000);
1794 WREG32((0x2c20 + j), 0x00000000);
1795 WREG32((0x2c24 + j), 0x00000000);
1796 }
1797
1798 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1799
1800 /* Setup tiling */
1801 tiling_config = 0;
1802 ramcfg = RREG32(RAMCFG);
1803 switch (rdev->config.r600.max_tile_pipes) {
1804 case 1:
1805 tiling_config |= PIPE_TILING(0);
1806 break;
1807 case 2:
1808 tiling_config |= PIPE_TILING(1);
1809 break;
1810 case 4:
1811 tiling_config |= PIPE_TILING(2);
1812 break;
1813 case 8:
1814 tiling_config |= PIPE_TILING(3);
1815 break;
1816 default:
1817 break;
1818 }
d03f5d59 1819 rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
961fb597 1820 rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
3ce0a23d 1821 tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
881fe6c1 1822 tiling_config |= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
416a2bd2 1823
3ce0a23d
JG		 1824 tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1825 if (tmp > 3) {
1826 tiling_config |= ROW_TILING(3);
1827 tiling_config |= SAMPLE_SPLIT(3);
1828 } else {
1829 tiling_config |= ROW_TILING(tmp);
1830 tiling_config |= SAMPLE_SPLIT(tmp);
1831 }
1832 tiling_config |= BANK_SWAPS(1);
d03f5d59
AD		 1833
1834 cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
416a2bd2
AD		 1835 tmp = R6XX_MAX_BACKENDS -
1836 r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK);
1837 if (tmp < rdev->config.r600.max_backends) {
1838 rdev->config.r600.max_backends = tmp;
1839 }
1840
1841 cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
1842 tmp = R6XX_MAX_PIPES -
1843 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK);
1844 if (tmp < rdev->config.r600.max_pipes) {
1845 rdev->config.r600.max_pipes = tmp;
1846 }
1847 tmp = R6XX_MAX_SIMDS -
1848 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
1849 if (tmp < rdev->config.r600.max_simds) {
1850 rdev->config.r600.max_simds = tmp;
1851 }
1852
1853 disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
1854 tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1855 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
1856 R6XX_MAX_BACKENDS, disabled_rb_mask);
1857 tiling_config |= tmp << 16;
1858 rdev->config.r600.backend_map = tmp;
1859
e7aeeba6 1860 rdev->config.r600.tile_config = tiling_config;
3ce0a23d
JG		 1861 WREG32(GB_TILING_CONFIG, tiling_config);
1862 WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1863 WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
4d75658b 1864 WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
3ce0a23d 1865
d03f5d59 1866 tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
3ce0a23d
JG		 1867 WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1868 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1869
1870 /* Setup some CP states */
1871 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1872 WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1873
1874 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1875 SYNC_WALKER | SYNC_ALIGNER));
1876 /* Setup various GPU states */
1877 if (rdev->family == CHIP_RV670)
1878 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1879
1880 tmp = RREG32(SX_DEBUG_1);
1881 tmp |= SMX_EVENT_RELEASE;
1882 if ((rdev->family > CHIP_R600))
1883 tmp |= ENABLE_NEW_SMX_ADDRESS;
1884 WREG32(SX_DEBUG_1, tmp);
1885
1886 if (((rdev->family) == CHIP_R600) ||
1887 ((rdev->family) == CHIP_RV630) ||
1888 ((rdev->family) == CHIP_RV610) ||
1889 ((rdev->family) == CHIP_RV620) ||
ee59f2b4
AD		 1890 ((rdev->family) == CHIP_RS780) ||
1891 ((rdev->family) == CHIP_RS880)) {
3ce0a23d
JG		 1892 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1893 } else {
1894 WREG32(DB_DEBUG, 0);
1895 }
1896 WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1897 DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1898
1899 WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1900 WREG32(VGT_NUM_INSTANCES, 0);
1901
1902 WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1903 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1904
1905 tmp = RREG32(SQ_MS_FIFO_SIZES);
1906 if (((rdev->family) == CHIP_RV610) ||
1907 ((rdev->family) == CHIP_RV620) ||
ee59f2b4
AD		 1908 ((rdev->family) == CHIP_RS780) ||
1909 ((rdev->family) == CHIP_RS880)) {
3ce0a23d
JG		 1910 tmp = (CACHE_FIFO_SIZE(0xa) |
1911 FETCH_FIFO_HIWATER(0xa) |
1912 DONE_FIFO_HIWATER(0xe0) |
1913 ALU_UPDATE_FIFO_HIWATER(0x8));
1914 } else if (((rdev->family) == CHIP_R600) ||
1915 ((rdev->family) == CHIP_RV630)) {
1916 tmp &= ~DONE_FIFO_HIWATER(0xff);
1917 tmp |= DONE_FIFO_HIWATER(0x4);
1918 }
1919 WREG32(SQ_MS_FIFO_SIZES, tmp);
1920
1921 /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1922 * should be adjusted as needed by the 2D/3D drivers. This just sets default values
1923 */
1924 sq_config = RREG32(SQ_CONFIG);
1925 sq_config &= ~(PS_PRIO(3) |
1926 VS_PRIO(3) |
1927 GS_PRIO(3) |
1928 ES_PRIO(3));
1929 sq_config |= (DX9_CONSTS |
1930 VC_ENABLE |
1931 PS_PRIO(0) |
1932 VS_PRIO(1) |
1933 GS_PRIO(2) |
1934 ES_PRIO(3));
1935
1936 if ((rdev->family) == CHIP_R600) {
1937 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1938 NUM_VS_GPRS(124) |
1939 NUM_CLAUSE_TEMP_GPRS(4));
1940 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1941 NUM_ES_GPRS(0));
1942 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1943 NUM_VS_THREADS(48) |
1944 NUM_GS_THREADS(4) |
1945 NUM_ES_THREADS(4));
1946 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1947 NUM_VS_STACK_ENTRIES(128));
1948 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1949 NUM_ES_STACK_ENTRIES(0));
1950 } else if (((rdev->family) == CHIP_RV610) ||
1951 ((rdev->family) == CHIP_RV620) ||
ee59f2b4
AD		 1952 ((rdev->family) == CHIP_RS780) ||
1953 ((rdev->family) == CHIP_RS880)) {
3ce0a23d
JG		 1954 /* no vertex cache */
1955 sq_config &= ~VC_ENABLE;
1956
1957 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1958 NUM_VS_GPRS(44) |
1959 NUM_CLAUSE_TEMP_GPRS(2));
1960 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1961 NUM_ES_GPRS(17));
1962 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1963 NUM_VS_THREADS(78) |
1964 NUM_GS_THREADS(4) |
1965 NUM_ES_THREADS(31));
1966 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1967 NUM_VS_STACK_ENTRIES(40));
1968 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1969 NUM_ES_STACK_ENTRIES(16));
1970 } else if (((rdev->family) == CHIP_RV630) ||
1971 ((rdev->family) == CHIP_RV635)) {
1972 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1973 NUM_VS_GPRS(44) |
1974 NUM_CLAUSE_TEMP_GPRS(2));
1975 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1976 NUM_ES_GPRS(18));
1977 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1978 NUM_VS_THREADS(78) |
1979 NUM_GS_THREADS(4) |
1980 NUM_ES_THREADS(31));
1981 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1982 NUM_VS_STACK_ENTRIES(40));
1983 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1984 NUM_ES_STACK_ENTRIES(16));
1985 } else if ((rdev->family) == CHIP_RV670) {
1986 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1987 NUM_VS_GPRS(44) |
1988 NUM_CLAUSE_TEMP_GPRS(2));
1989 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1990 NUM_ES_GPRS(17));
1991 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1992 NUM_VS_THREADS(78) |
1993 NUM_GS_THREADS(4) |
1994 NUM_ES_THREADS(31));
1995 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
1996 NUM_VS_STACK_ENTRIES(64));
1997 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
1998 NUM_ES_STACK_ENTRIES(64));
1999 }
2000
2001 WREG32(SQ_CONFIG, sq_config);
2002 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2003 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2004 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2005 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2006 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2007
2008 if (((rdev->family) == CHIP_RV610) ||
2009 ((rdev->family) == CHIP_RV620) ||
ee59f2b4
AD		 2010 ((rdev->family) == CHIP_RS780) ||
2011 ((rdev->family) == CHIP_RS880)) {
3ce0a23d
JG		 2012 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
2013 } else {
2014 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
2015 }
2016
2017 /* More default values. 2D/3D driver should adjust as needed */
2018 WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
2019 S1_X(0x4) | S1_Y(0xc)));
2020 WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
2021 S1_X(0x2) | S1_Y(0x2) |
2022 S2_X(0xa) | S2_Y(0x6) |
2023 S3_X(0x6) | S3_Y(0xa)));
2024 WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
2025 S1_X(0x4) | S1_Y(0xc) |
2026 S2_X(0x1) | S2_Y(0x6) |
2027 S3_X(0xa) | S3_Y(0xe)));
2028 WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
2029 S5_X(0x0) | S5_Y(0x0) |
2030 S6_X(0xb) | S6_Y(0x4) |
2031 S7_X(0x7) | S7_Y(0x8)));
2032
2033 WREG32(VGT_STRMOUT_EN, 0);
2034 tmp = rdev->config.r600.max_pipes * 16;
2035 switch (rdev->family) {
2036 case CHIP_RV610:
3ce0a23d 2037 case CHIP_RV620:
ee59f2b4
AD		 2038 case CHIP_RS780:
2039 case CHIP_RS880:
3ce0a23d
JG		 2040 tmp += 32;
2041 break;
2042 case CHIP_RV670:
2043 tmp += 128;
2044 break;
2045 default:
2046 break;
2047 }
2048 if (tmp > 256) {
2049 tmp = 256;
2050 }
2051 WREG32(VGT_ES_PER_GS, 128);
2052 WREG32(VGT_GS_PER_ES, tmp);
2053 WREG32(VGT_GS_PER_VS, 2);
2054 WREG32(VGT_GS_VERTEX_REUSE, 16);
2055
2056 /* more default values. 2D/3D driver should adjust as needed */
2057 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2058 WREG32(VGT_STRMOUT_EN, 0);
2059 WREG32(SX_MISC, 0);
2060 WREG32(PA_SC_MODE_CNTL, 0);
2061 WREG32(PA_SC_AA_CONFIG, 0);
2062 WREG32(PA_SC_LINE_STIPPLE, 0);
2063 WREG32(SPI_INPUT_Z, 0);
2064 WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
2065 WREG32(CB_COLOR7_FRAG, 0);
2066
2067 /* Clear render buffer base addresses */
2068 WREG32(CB_COLOR0_BASE, 0);
2069 WREG32(CB_COLOR1_BASE, 0);
2070 WREG32(CB_COLOR2_BASE, 0);
2071 WREG32(CB_COLOR3_BASE, 0);
2072 WREG32(CB_COLOR4_BASE, 0);
2073 WREG32(CB_COLOR5_BASE, 0);
2074 WREG32(CB_COLOR6_BASE, 0);
2075 WREG32(CB_COLOR7_BASE, 0);
2076 WREG32(CB_COLOR7_FRAG, 0);
2077
2078 switch (rdev->family) {
2079 case CHIP_RV610:
3ce0a23d 2080 case CHIP_RV620:
ee59f2b4
AD		 2081 case CHIP_RS780:
2082 case CHIP_RS880:
3ce0a23d
JG		 2083 tmp = TC_L2_SIZE(8);
2084 break;
2085 case CHIP_RV630:
2086 case CHIP_RV635:
2087 tmp = TC_L2_SIZE(4);
2088 break;
2089 case CHIP_R600:
2090 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
2091 break;
2092 default:
2093 tmp = TC_L2_SIZE(0);
2094 break;
2095 }
2096 WREG32(TC_CNTL, tmp);
2097
2098 tmp = RREG32(HDP_HOST_PATH_CNTL);
2099 WREG32(HDP_HOST_PATH_CNTL, tmp);
2100
2101 tmp = RREG32(ARB_POP);
2102 tmp |= ENABLE_TC128;
2103 WREG32(ARB_POP, tmp);
2104
2105 WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
2106 WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
2107 NUM_CLIP_SEQ(3)));
2108 WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
b866d133 2109 WREG32(VC_ENHANCE, 0);
3ce0a23d
JG		 2110}
2111
2112
771fe6b9
JG		 2113/*
2114 * Indirect registers accessor
2115 */
3ce0a23d
JG		 2116u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
2117{
2118 u32 r;
2119
2120 WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2121 (void)RREG32(PCIE_PORT_INDEX);
2122 r = RREG32(PCIE_PORT_DATA);
2123 return r;
2124}
2125
2126void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
2127{
2128 WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2129 (void)RREG32(PCIE_PORT_INDEX);
2130 WREG32(PCIE_PORT_DATA, (v));
2131 (void)RREG32(PCIE_PORT_DATA);
2132}
2133
3ce0a23d
JG		 2134/*
2135 * CP & Ring
2136 */
2137void r600_cp_stop(struct radeon_device *rdev)
2138{
53595338 2139 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3ce0a23d 2140 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
724c80e1 2141 WREG32(SCRATCH_UMSK, 0);
4d75658b 2142 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3ce0a23d
JG		 2143}
2144
d8f60cfc 2145int r600_init_microcode(struct radeon_device *rdev)
3ce0a23d
JG		 2146{
2147 struct platform_device *pdev;
2148 const char *chip_name;
d8f60cfc 2149 const char *rlc_chip_name;
66229b20
AD		 2150 const char *smc_chip_name = "RV770";
2151 size_t pfp_req_size, me_req_size, rlc_req_size, smc_req_size = 0;
3ce0a23d
JG		 2152 char fw_name[30];
2153 int err;
2154
2155 DRM_DEBUG("\n");
2156
2157 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
2158 err = IS_ERR(pdev);
2159 if (err) {
2160 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
2161 return -EINVAL;
2162 }
2163
2164 switch (rdev->family) {
d8f60cfc
AD		 2165 case CHIP_R600:
2166 chip_name = "R600";
2167 rlc_chip_name = "R600";
2168 break;
2169 case CHIP_RV610:
2170 chip_name = "RV610";
2171 rlc_chip_name = "R600";
2172 break;
2173 case CHIP_RV630:
2174 chip_name = "RV630";
2175 rlc_chip_name = "R600";
2176 break;
2177 case CHIP_RV620:
2178 chip_name = "RV620";
2179 rlc_chip_name = "R600";
2180 break;
2181 case CHIP_RV635:
2182 chip_name = "RV635";
2183 rlc_chip_name = "R600";
2184 break;
2185 case CHIP_RV670:
2186 chip_name = "RV670";
2187 rlc_chip_name = "R600";
2188 break;
3ce0a23d 2189 case CHIP_RS780:
d8f60cfc
AD		 2190 case CHIP_RS880:
2191 chip_name = "RS780";
2192 rlc_chip_name = "R600";
2193 break;
2194 case CHIP_RV770:
2195 chip_name = "RV770";
2196 rlc_chip_name = "R700";
66229b20
AD		 2197 smc_chip_name = "RV770";
2198 smc_req_size = ALIGN(RV770_SMC_UCODE_SIZE, 4);
d8f60cfc 2199 break;
3ce0a23d 2200 case CHIP_RV730:
d8f60cfc
AD		 2201 chip_name = "RV730";
2202 rlc_chip_name = "R700";
66229b20
AD		 2203 smc_chip_name = "RV730";
2204 smc_req_size = ALIGN(RV730_SMC_UCODE_SIZE, 4);
d8f60cfc
AD		 2205 break;
2206 case CHIP_RV710:
2207 chip_name = "RV710";
2208 rlc_chip_name = "R700";
66229b20
AD		 2209 smc_chip_name = "RV710";
2210 smc_req_size = ALIGN(RV710_SMC_UCODE_SIZE, 4);
2211 break;
2212 case CHIP_RV740:
2213 chip_name = "RV730";
2214 rlc_chip_name = "R700";
2215 smc_chip_name = "RV740";
2216 smc_req_size = ALIGN(RV740_SMC_UCODE_SIZE, 4);
d8f60cfc 2217 break;
fe251e2f
AD		 2218 case CHIP_CEDAR:
2219 chip_name = "CEDAR";
45f9a39b 2220 rlc_chip_name = "CEDAR";
dc50ba7f
AD		 2221 smc_chip_name = "CEDAR";
2222 smc_req_size = ALIGN(CEDAR_SMC_UCODE_SIZE, 4);
fe251e2f
AD		 2223 break;
2224 case CHIP_REDWOOD:
2225 chip_name = "REDWOOD";
45f9a39b 2226 rlc_chip_name = "REDWOOD";
dc50ba7f
AD		 2227 smc_chip_name = "REDWOOD";
2228 smc_req_size = ALIGN(REDWOOD_SMC_UCODE_SIZE, 4);
fe251e2f
AD		 2229 break;
2230 case CHIP_JUNIPER:
2231 chip_name = "JUNIPER";
45f9a39b 2232 rlc_chip_name = "JUNIPER";
dc50ba7f
AD		 2233 smc_chip_name = "JUNIPER";
2234 smc_req_size = ALIGN(JUNIPER_SMC_UCODE_SIZE, 4);
fe251e2f
AD		 2235 break;
2236 case CHIP_CYPRESS:
2237 case CHIP_HEMLOCK:
2238 chip_name = "CYPRESS";
45f9a39b 2239 rlc_chip_name = "CYPRESS";
dc50ba7f
AD		 2240 smc_chip_name = "CYPRESS";
2241 smc_req_size = ALIGN(CYPRESS_SMC_UCODE_SIZE, 4);
fe251e2f 2242 break;
439bd6cd
AD		 2243 case CHIP_PALM:
2244 chip_name = "PALM";
2245 rlc_chip_name = "SUMO";
2246 break;
d5c5a72f
AD		 2247 case CHIP_SUMO:
2248 chip_name = "SUMO";
2249 rlc_chip_name = "SUMO";
2250 break;
2251 case CHIP_SUMO2:
2252 chip_name = "SUMO2";
2253 rlc_chip_name = "SUMO";
2254 break;
3ce0a23d
JG		 2255 default: BUG();
2256 }
2257
fe251e2f
AD		 2258 if (rdev->family >= CHIP_CEDAR) {
2259 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
2260 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
45f9a39b 2261 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
fe251e2f 2262 } else if (rdev->family >= CHIP_RV770) {
3ce0a23d
JG		 2263 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
2264 me_req_size = R700_PM4_UCODE_SIZE * 4;
d8f60cfc 2265 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
3ce0a23d 2266 } else {
138e4e16
AD		 2267 pfp_req_size = R600_PFP_UCODE_SIZE * 4;
2268 me_req_size = R600_PM4_UCODE_SIZE * 12;
2269 rlc_req_size = R600_RLC_UCODE_SIZE * 4;
3ce0a23d
JG		 2270 }
2271
d8f60cfc 2272 DRM_INFO("Loading %s Microcode\n", chip_name);
3ce0a23d
JG		 2273
2274 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2275 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
2276 if (err)
2277 goto out;
2278 if (rdev->pfp_fw->size != pfp_req_size) {
2279 printk(KERN_ERR
2280 "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2281 rdev->pfp_fw->size, fw_name);
2282 err = -EINVAL;
2283 goto out;
2284 }
2285
2286 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2287 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
2288 if (err)
2289 goto out;
2290 if (rdev->me_fw->size != me_req_size) {
2291 printk(KERN_ERR
2292 "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2293 rdev->me_fw->size, fw_name);
2294 err = -EINVAL;
2295 }
d8f60cfc
AD		 2296
2297 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
2298 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
2299 if (err)
2300 goto out;
2301 if (rdev->rlc_fw->size != rlc_req_size) {
2302 printk(KERN_ERR
2303 "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
2304 rdev->rlc_fw->size, fw_name);
2305 err = -EINVAL;
2306 }
2307
dc50ba7f 2308 if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_HEMLOCK)) {
66229b20
AD		 2309 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", smc_chip_name);
2310 err = request_firmware(&rdev->smc_fw, fw_name, &pdev->dev);
2311 if (err)
2312 goto out;
2313 if (rdev->smc_fw->size != smc_req_size) {
2314 printk(KERN_ERR
2315 "smc: Bogus length %zu in firmware \"%s\"\n",
2316 rdev->smc_fw->size, fw_name);
2317 err = -EINVAL;
2318 }
2319 }
2320
3ce0a23d
JG		 2321out:
2322 platform_device_unregister(pdev);
2323
2324 if (err) {
2325 if (err != -EINVAL)
2326 printk(KERN_ERR
2327 "r600_cp: Failed to load firmware \"%s\"\n",
2328 fw_name);
2329 release_firmware(rdev->pfp_fw);
2330 rdev->pfp_fw = NULL;
2331 release_firmware(rdev->me_fw);
2332 rdev->me_fw = NULL;
d8f60cfc
AD		 2333 release_firmware(rdev->rlc_fw);
2334 rdev->rlc_fw = NULL;
66229b20
AD		 2335 release_firmware(rdev->smc_fw);
2336 rdev->smc_fw = NULL;
3ce0a23d
JG		 2337 }
2338 return err;
2339}
2340
2341static int r600_cp_load_microcode(struct radeon_device *rdev)
2342{
2343 const __be32 *fw_data;
2344 int i;
2345
2346 if (!rdev->me_fw || !rdev->pfp_fw)
2347 return -EINVAL;
2348
2349 r600_cp_stop(rdev);
2350
4eace7fd
CC		 2351 WREG32(CP_RB_CNTL,
2352#ifdef __BIG_ENDIAN
2353 BUF_SWAP_32BIT |
2354#endif
2355 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
3ce0a23d
JG		 2356
2357 /* Reset cp */
2358 WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2359 RREG32(GRBM_SOFT_RESET);
2360 mdelay(15);
2361 WREG32(GRBM_SOFT_RESET, 0);
2362
2363 WREG32(CP_ME_RAM_WADDR, 0);
2364
2365 fw_data = (const __be32 *)rdev->me_fw->data;
2366 WREG32(CP_ME_RAM_WADDR, 0);
138e4e16 2367 for (i = 0; i < R600_PM4_UCODE_SIZE * 3; i++)
3ce0a23d
JG		 2368 WREG32(CP_ME_RAM_DATA,
2369 be32_to_cpup(fw_data++));
2370
2371 fw_data = (const __be32 *)rdev->pfp_fw->data;
2372 WREG32(CP_PFP_UCODE_ADDR, 0);
138e4e16 2373 for (i = 0; i < R600_PFP_UCODE_SIZE; i++)
3ce0a23d
JG		 2374 WREG32(CP_PFP_UCODE_DATA,
2375 be32_to_cpup(fw_data++));
2376
2377 WREG32(CP_PFP_UCODE_ADDR, 0);
2378 WREG32(CP_ME_RAM_WADDR, 0);
2379 WREG32(CP_ME_RAM_RADDR, 0);
2380 return 0;
2381}
2382
2383int r600_cp_start(struct radeon_device *rdev)
2384{
e32eb50d 2385 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3ce0a23d
JG		 2386 int r;
2387 uint32_t cp_me;
2388
e32eb50d 2389 r = radeon_ring_lock(rdev, ring, 7);
3ce0a23d
JG		 2390 if (r) {
2391 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2392 return r;
2393 }
e32eb50d
CK		 2394 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2395 radeon_ring_write(ring, 0x1);
7e7b41d2 2396 if (rdev->family >= CHIP_RV770) {
e32eb50d
CK		 2397 radeon_ring_write(ring, 0x0);
2398 radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
fe251e2f 2399 } else {
e32eb50d
CK		 2400 radeon_ring_write(ring, 0x3);
2401 radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
3ce0a23d 2402 }
e32eb50d
CK		 2403 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2404 radeon_ring_write(ring, 0);
2405 radeon_ring_write(ring, 0);
2406 radeon_ring_unlock_commit(rdev, ring);
3ce0a23d
JG		 2407
2408 cp_me = 0xff;
2409 WREG32(R_0086D8_CP_ME_CNTL, cp_me);
2410 return 0;
2411}
2412
2413int r600_cp_resume(struct radeon_device *rdev)
2414{
e32eb50d 2415 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3ce0a23d
JG		 2416 u32 tmp;
2417 u32 rb_bufsz;
2418 int r;
2419
2420 /* Reset cp */
2421 WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2422 RREG32(GRBM_SOFT_RESET);
2423 mdelay(15);
2424 WREG32(GRBM_SOFT_RESET, 0);
2425
2426 /* Set ring buffer size */
e32eb50d 2427 rb_bufsz = drm_order(ring->ring_size / 8);
724c80e1 2428 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3ce0a23d 2429#ifdef __BIG_ENDIAN
d6f28938 2430 tmp |= BUF_SWAP_32BIT;
3ce0a23d 2431#endif
d6f28938 2432 WREG32(CP_RB_CNTL, tmp);
15d3332f 2433 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3ce0a23d
JG		 2434
2435 /* Set the write pointer delay */
2436 WREG32(CP_RB_WPTR_DELAY, 0);
2437
2438 /* Initialize the ring buffer's read and write pointers */
3ce0a23d
JG		 2439 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2440 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2441 ring->wptr = 0;
2442 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1
AD		 2443
2444 /* set the wb address whether it's enabled or not */
4eace7fd 2445 WREG32(CP_RB_RPTR_ADDR,
4eace7fd 2446 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2447 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2448 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2449
2450 if (rdev->wb.enabled)
2451 WREG32(SCRATCH_UMSK, 0xff);
2452 else {
2453 tmp |= RB_NO_UPDATE;
2454 WREG32(SCRATCH_UMSK, 0);
2455 }
2456
3ce0a23d
JG		 2457 mdelay(1);
2458 WREG32(CP_RB_CNTL, tmp);
2459
e32eb50d 2460 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3ce0a23d
JG		 2461 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2462
e32eb50d 2463 ring->rptr = RREG32(CP_RB_RPTR);
3ce0a23d
JG		 2464
2465 r600_cp_start(rdev);
e32eb50d 2466 ring->ready = true;
f712812e 2467 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3ce0a23d 2468 if (r) {
e32eb50d 2469 ring->ready = false;
3ce0a23d
JG		 2470 return r;
2471 }
2472 return 0;
2473}
2474
e32eb50d 2475void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)
3ce0a23d
JG		 2476{
2477 u32 rb_bufsz;
45df6803 2478 int r;
3ce0a23d
JG		 2479
2480 /* Align ring size */
2481 rb_bufsz = drm_order(ring_size / 8);
2482 ring_size = (1 << (rb_bufsz + 1)) * 4;
e32eb50d
CK		 2483 ring->ring_size = ring_size;
2484 ring->align_mask = 16 - 1;
45df6803 2485
89d35807
AD		 2486 if (radeon_ring_supports_scratch_reg(rdev, ring)) {
2487 r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
2488 if (r) {
2489 DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
2490 ring->rptr_save_reg = 0;
2491 }
45df6803 2492 }
3ce0a23d
JG		 2493}
2494
655efd3d
JG		 2495void r600_cp_fini(struct radeon_device *rdev)
2496{
45df6803 2497 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
655efd3d 2498 r600_cp_stop(rdev);
45df6803
CK		 2499 radeon_ring_fini(rdev, ring);
2500 radeon_scratch_free(rdev, ring->rptr_save_reg);
655efd3d
JG		 2501}
2502
4d75658b
AD		 2503/*
2504 * DMA
2505 * Starting with R600, the GPU has an asynchronous
2506 * DMA engine. The programming model is very similar
2507 * to the 3D engine (ring buffer, IBs, etc.), but the
2508 * DMA controller has it's own packet format that is
2509 * different form the PM4 format used by the 3D engine.
2510 * It supports copying data, writing embedded data,
2511 * solid fills, and a number of other things. It also
2512 * has support for tiling/detiling of buffers.
2513 */
2514/**
2515 * r600_dma_stop - stop the async dma engine
2516 *
2517 * @rdev: radeon_device pointer
2518 *
2519 * Stop the async dma engine (r6xx-evergreen).
2520 */
2521void r600_dma_stop(struct radeon_device *rdev)
2522{
2523 u32 rb_cntl = RREG32(DMA_RB_CNTL);
2524
2525 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2526
2527 rb_cntl &= ~DMA_RB_ENABLE;
2528 WREG32(DMA_RB_CNTL, rb_cntl);
2529
2530 rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
2531}
2532
2533/**
2534 * r600_dma_resume - setup and start the async dma engine
2535 *
2536 * @rdev: radeon_device pointer
2537 *
2538 * Set up the DMA ring buffer and enable it. (r6xx-evergreen).
2539 * Returns 0 for success, error for failure.
2540 */
2541int r600_dma_resume(struct radeon_device *rdev)
2542{
2543 struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
b3dfcb20 2544 u32 rb_cntl, dma_cntl, ib_cntl;
4d75658b
AD		 2545 u32 rb_bufsz;
2546 int r;
2547
2548 /* Reset dma */
2549 if (rdev->family >= CHIP_RV770)
2550 WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA);
2551 else
2552 WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA);
2553 RREG32(SRBM_SOFT_RESET);
2554 udelay(50);
2555 WREG32(SRBM_SOFT_RESET, 0);
2556
2557 WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL, 0);
2558 WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
2559
2560 /* Set ring buffer size in dwords */
2561 rb_bufsz = drm_order(ring->ring_size / 4);
2562 rb_cntl = rb_bufsz << 1;
2563#ifdef __BIG_ENDIAN
2564 rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
2565#endif
2566 WREG32(DMA_RB_CNTL, rb_cntl);
2567
2568 /* Initialize the ring buffer's read and write pointers */
2569 WREG32(DMA_RB_RPTR, 0);
2570 WREG32(DMA_RB_WPTR, 0);
2571
2572 /* set the wb address whether it's enabled or not */
2573 WREG32(DMA_RB_RPTR_ADDR_HI,
2574 upper_32_bits(rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFF);
2575 WREG32(DMA_RB_RPTR_ADDR_LO,
2576 ((rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFFFFFFFC));
2577
2578 if (rdev->wb.enabled)
2579 rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
2580
2581 WREG32(DMA_RB_BASE, ring->gpu_addr >> 8);
2582
2583 /* enable DMA IBs */
b3dfcb20
MD		 2584 ib_cntl = DMA_IB_ENABLE;
2585#ifdef __BIG_ENDIAN
2586 ib_cntl |= DMA_IB_SWAP_ENABLE;
2587#endif
2588 WREG32(DMA_IB_CNTL, ib_cntl);
4d75658b
AD		 2589
2590 dma_cntl = RREG32(DMA_CNTL);
2591 dma_cntl &= ~CTXEMPTY_INT_ENABLE;
2592 WREG32(DMA_CNTL, dma_cntl);
2593
2594 if (rdev->family >= CHIP_RV770)
2595 WREG32(DMA_MODE, 1);
2596
2597 ring->wptr = 0;
2598 WREG32(DMA_RB_WPTR, ring->wptr << 2);
2599
2600 ring->rptr = RREG32(DMA_RB_RPTR) >> 2;
2601
2602 WREG32(DMA_RB_CNTL, rb_cntl | DMA_RB_ENABLE);
2603
2604 ring->ready = true;
2605
2606 r = radeon_ring_test(rdev, R600_RING_TYPE_DMA_INDEX, ring);
2607 if (r) {
2608 ring->ready = false;
2609 return r;
2610 }
2611
2612 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
2613
2614 return 0;
2615}
2616
2617/**
2618 * r600_dma_fini - tear down the async dma engine
2619 *
2620 * @rdev: radeon_device pointer
2621 *
2622 * Stop the async dma engine and free the ring (r6xx-evergreen).
2623 */
2624void r600_dma_fini(struct radeon_device *rdev)
2625{
2626 r600_dma_stop(rdev);
2627 radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
2628}
3ce0a23d 2629
f2ba57b5
CK		 2630/*
2631 * UVD
2632 */
2633int r600_uvd_rbc_start(struct radeon_device *rdev)
2634{
2635 struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2636 uint64_t rptr_addr;
2637 uint32_t rb_bufsz, tmp;
2638 int r;
2639
2640 rptr_addr = rdev->wb.gpu_addr + R600_WB_UVD_RPTR_OFFSET;
2641
2642 if (upper_32_bits(rptr_addr) != upper_32_bits(ring->gpu_addr)) {
2643 DRM_ERROR("UVD ring and rptr not in the same 4GB segment!\n");
2644 return -EINVAL;
2645 }
2646
2647 /* force RBC into idle state */
2648 WREG32(UVD_RBC_RB_CNTL, 0x11010101);
2649
2650 /* Set the write pointer delay */
2651 WREG32(UVD_RBC_RB_WPTR_CNTL, 0);
2652
2653 /* set the wb address */
2654 WREG32(UVD_RBC_RB_RPTR_ADDR, rptr_addr >> 2);
2655
2656 /* programm the 4GB memory segment for rptr and ring buffer */
2657 WREG32(UVD_LMI_EXT40_ADDR, upper_32_bits(rptr_addr) |
2658 (0x7 << 16) | (0x1 << 31));
2659
2660 /* Initialize the ring buffer's read and write pointers */
2661 WREG32(UVD_RBC_RB_RPTR, 0x0);
2662
2663 ring->wptr = ring->rptr = RREG32(UVD_RBC_RB_RPTR);
2664 WREG32(UVD_RBC_RB_WPTR, ring->wptr);
2665
2666 /* set the ring address */
2667 WREG32(UVD_RBC_RB_BASE, ring->gpu_addr);
2668
2669 /* Set ring buffer size */
2670 rb_bufsz = drm_order(ring->ring_size);
2671 rb_bufsz = (0x1 << 8) | rb_bufsz;
2672 WREG32(UVD_RBC_RB_CNTL, rb_bufsz);
2673
2674 ring->ready = true;
2675 r = radeon_ring_test(rdev, R600_RING_TYPE_UVD_INDEX, ring);
2676 if (r) {
2677 ring->ready = false;
2678 return r;
2679 }
2680
2681 r = radeon_ring_lock(rdev, ring, 10);
2682 if (r) {
2683 DRM_ERROR("radeon: ring failed to lock UVD ring (%d).\n", r);
2684 return r;
2685 }
2686
2687 tmp = PACKET0(UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL, 0);
2688 radeon_ring_write(ring, tmp);
2689 radeon_ring_write(ring, 0xFFFFF);
2690
2691 tmp = PACKET0(UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL, 0);
2692 radeon_ring_write(ring, tmp);
2693 radeon_ring_write(ring, 0xFFFFF);
2694
2695 tmp = PACKET0(UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL, 0);
2696 radeon_ring_write(ring, tmp);
2697 radeon_ring_write(ring, 0xFFFFF);
2698
2699 /* Clear timeout status bits */
2700 radeon_ring_write(ring, PACKET0(UVD_SEMA_TIMEOUT_STATUS, 0));
2701 radeon_ring_write(ring, 0x8);
2702
2703 radeon_ring_write(ring, PACKET0(UVD_SEMA_CNTL, 0));
03708b05 2704 radeon_ring_write(ring, 3);
f2ba57b5
CK		 2705
2706 radeon_ring_unlock_commit(rdev, ring);
2707
2708 return 0;
2709}
2710
2711void r600_uvd_rbc_stop(struct radeon_device *rdev)
2712{
2713 struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2714
2715 /* force RBC into idle state */
2716 WREG32(UVD_RBC_RB_CNTL, 0x11010101);
2717 ring->ready = false;
2718}
2719
2720int r600_uvd_init(struct radeon_device *rdev)
2721{
2722 int i, j, r;
9b1be4dc
AD		 2723 /* disable byte swapping */
2724 u32 lmi_swap_cntl = 0;
2725 u32 mp_swap_cntl = 0;
f2ba57b5 2726
b05e9e4c
CK		 2727 /* raise clocks while booting up the VCPU */
2728 radeon_set_uvd_clocks(rdev, 53300, 40000);
2729
f2ba57b5
CK		 2730 /* disable clock gating */
2731 WREG32(UVD_CGC_GATE, 0);
2732
2733 /* disable interupt */
2734 WREG32_P(UVD_MASTINT_EN, 0, ~(1 << 1));
2735
2736 /* put LMI, VCPU, RBC etc... into reset */
2737 WREG32(UVD_SOFT_RESET, LMI_SOFT_RESET | VCPU_SOFT_RESET |
2738 LBSI_SOFT_RESET | RBC_SOFT_RESET | CSM_SOFT_RESET |
2739 CXW_SOFT_RESET | TAP_SOFT_RESET | LMI_UMC_SOFT_RESET);
2740 mdelay(5);
2741
2742 /* take UVD block out of reset */
2743 WREG32_P(SRBM_SOFT_RESET, 0, ~SOFT_RESET_UVD);
2744 mdelay(5);
2745
2746 /* initialize UVD memory controller */
2747 WREG32(UVD_LMI_CTRL, 0x40 | (1 << 8) | (1 << 13) |
2748 (1 << 21) | (1 << 9) | (1 << 20));
2749
9b1be4dc
AD		 2750#ifdef __BIG_ENDIAN
2751 /* swap (8 in 32) RB and IB */
2752 lmi_swap_cntl = 0xa;
2753 mp_swap_cntl = 0;
2754#endif
2755 WREG32(UVD_LMI_SWAP_CNTL, lmi_swap_cntl);
2756 WREG32(UVD_MP_SWAP_CNTL, mp_swap_cntl);
f2ba57b5
CK		 2757
2758 WREG32(UVD_MPC_SET_MUXA0, 0x40c2040);
2759 WREG32(UVD_MPC_SET_MUXA1, 0x0);
2760 WREG32(UVD_MPC_SET_MUXB0, 0x40c2040);
2761 WREG32(UVD_MPC_SET_MUXB1, 0x0);
2762 WREG32(UVD_MPC_SET_ALU, 0);
2763 WREG32(UVD_MPC_SET_MUX, 0x88);
2764
2765 /* Stall UMC */
2766 WREG32_P(UVD_LMI_CTRL2, 1 << 8, ~(1 << 8));
2767 WREG32_P(UVD_RB_ARB_CTRL, 1 << 3, ~(1 << 3));
2768
2769 /* take all subblocks out of reset, except VCPU */
2770 WREG32(UVD_SOFT_RESET, VCPU_SOFT_RESET);
2771 mdelay(5);
2772
2773 /* enable VCPU clock */
2774 WREG32(UVD_VCPU_CNTL, 1 << 9);
2775
2776 /* enable UMC */
2777 WREG32_P(UVD_LMI_CTRL2, 0, ~(1 << 8));
2778
2779 /* boot up the VCPU */
2780 WREG32(UVD_SOFT_RESET, 0);
2781 mdelay(10);
2782
2783 WREG32_P(UVD_RB_ARB_CTRL, 0, ~(1 << 3));
2784
2785 for (i = 0; i < 10; ++i) {
2786 uint32_t status;
2787 for (j = 0; j < 100; ++j) {
2788 status = RREG32(UVD_STATUS);
2789 if (status & 2)
2790 break;
2791 mdelay(10);
2792 }
2793 r = 0;
2794 if (status & 2)
2795 break;
2796
2797 DRM_ERROR("UVD not responding, trying to reset the VCPU!!!\n");
2798 WREG32_P(UVD_SOFT_RESET, VCPU_SOFT_RESET, ~VCPU_SOFT_RESET);
2799 mdelay(10);
2800 WREG32_P(UVD_SOFT_RESET, 0, ~VCPU_SOFT_RESET);
2801 mdelay(10);
2802 r = -1;
2803 }
b05e9e4c 2804
f2ba57b5
CK		 2805 if (r) {
2806 DRM_ERROR("UVD not responding, giving up!!!\n");
b05e9e4c 2807 radeon_set_uvd_clocks(rdev, 0, 0);
f2ba57b5
CK		 2808 return r;
2809 }
b05e9e4c 2810
f2ba57b5
CK		 2811 /* enable interupt */
2812 WREG32_P(UVD_MASTINT_EN, 3<<1, ~(3 << 1));
2813
2814 r = r600_uvd_rbc_start(rdev);
b05e9e4c
CK		 2815 if (!r)
2816 DRM_INFO("UVD initialized successfully.\n");
f2ba57b5 2817
b05e9e4c
CK		 2818 /* lower clocks again */
2819 radeon_set_uvd_clocks(rdev, 0, 0);
2820
2821 return r;
f2ba57b5
CK		 2822}
2823
3ce0a23d
JG		 2824/*
2825 * GPU scratch registers helpers function.
2826 */
2827void r600_scratch_init(struct radeon_device *rdev)
2828{
2829 int i;
2830
2831 rdev->scratch.num_reg = 7;
724c80e1 2832 rdev->scratch.reg_base = SCRATCH_REG0;
3ce0a23d
JG		 2833 for (i = 0; i < rdev->scratch.num_reg; i++) {
2834 rdev->scratch.free[i] = true;
724c80e1 2835 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3ce0a23d
JG		 2836 }
2837}
2838
e32eb50d 2839int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3ce0a23d
JG		 2840{
2841 uint32_t scratch;
2842 uint32_t tmp = 0;
8b25ed34 2843 unsigned i;
3ce0a23d
JG		 2844 int r;
2845
2846 r = radeon_scratch_get(rdev, &scratch);
2847 if (r) {
2848 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2849 return r;
2850 }
2851 WREG32(scratch, 0xCAFEDEAD);
e32eb50d 2852 r = radeon_ring_lock(rdev, ring, 3);
3ce0a23d 2853 if (r) {
8b25ed34 2854 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3ce0a23d
JG		 2855 radeon_scratch_free(rdev, scratch);
2856 return r;
2857 }
e32eb50d
CK		 2858 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2859 radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2860 radeon_ring_write(ring, 0xDEADBEEF);
2861 radeon_ring_unlock_commit(rdev, ring);
3ce0a23d
JG		 2862 for (i = 0; i < rdev->usec_timeout; i++) {
2863 tmp = RREG32(scratch);
2864 if (tmp == 0xDEADBEEF)
2865 break;
2866 DRM_UDELAY(1);
2867 }
2868 if (i < rdev->usec_timeout) {
8b25ed34 2869 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3ce0a23d 2870 } else {
bf852799 2871 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
8b25ed34 2872 ring->idx, scratch, tmp);
3ce0a23d
JG		 2873 r = -EINVAL;
2874 }
2875 radeon_scratch_free(rdev, scratch);
2876 return r;
2877}
2878
4d75658b
AD		 2879/**
2880 * r600_dma_ring_test - simple async dma engine test
2881 *
2882 * @rdev: radeon_device pointer
2883 * @ring: radeon_ring structure holding ring information
2884 *
2885 * Test the DMA engine by writing using it to write an
2886 * value to memory. (r6xx-SI).
2887 * Returns 0 for success, error for failure.
2888 */
2889int r600_dma_ring_test(struct radeon_device *rdev,
2890 struct radeon_ring *ring)
2891{
2892 unsigned i;
2893 int r;
2894 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
2895 u32 tmp;
2896
2897 if (!ptr) {
2898 DRM_ERROR("invalid vram scratch pointer\n");
2899 return -EINVAL;
2900 }
2901
2902 tmp = 0xCAFEDEAD;
2903 writel(tmp, ptr);
2904
2905 r = radeon_ring_lock(rdev, ring, 4);
2906 if (r) {
2907 DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
2908 return r;
2909 }
2910 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
2911 radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
2912 radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff);
2913 radeon_ring_write(ring, 0xDEADBEEF);
2914 radeon_ring_unlock_commit(rdev, ring);
2915
2916 for (i = 0; i < rdev->usec_timeout; i++) {
2917 tmp = readl(ptr);
2918 if (tmp == 0xDEADBEEF)
2919 break;
2920 DRM_UDELAY(1);
2921 }
2922
2923 if (i < rdev->usec_timeout) {
2924 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2925 } else {
2926 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
2927 ring->idx, tmp);
2928 r = -EINVAL;
2929 }
2930 return r;
2931}
2932
f2ba57b5
CK		 2933int r600_uvd_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2934{
2935 uint32_t tmp = 0;
2936 unsigned i;
2937 int r;
2938
2939 WREG32(UVD_CONTEXT_ID, 0xCAFEDEAD);
2940 r = radeon_ring_lock(rdev, ring, 3);
2941 if (r) {
2942 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n",
2943 ring->idx, r);
2944 return r;
2945 }
2946 radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0));
2947 radeon_ring_write(ring, 0xDEADBEEF);
2948 radeon_ring_unlock_commit(rdev, ring);
2949 for (i = 0; i < rdev->usec_timeout; i++) {
2950 tmp = RREG32(UVD_CONTEXT_ID);
2951 if (tmp == 0xDEADBEEF)
2952 break;
2953 DRM_UDELAY(1);
2954 }
2955
2956 if (i < rdev->usec_timeout) {
2957 DRM_INFO("ring test on %d succeeded in %d usecs\n",
2958 ring->idx, i);
2959 } else {
2960 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
2961 ring->idx, tmp);
2962 r = -EINVAL;
2963 }
2964 return r;
2965}
2966
4d75658b
AD		 2967/*
2968 * CP fences/semaphores
2969 */
2970
3ce0a23d
JG		 2971void r600_fence_ring_emit(struct radeon_device *rdev,
2972 struct radeon_fence *fence)
2973{
e32eb50d 2974 struct radeon_ring *ring = &rdev->ring[fence->ring];
7b1f2485 2975
d0f8a854 2976 if (rdev->wb.use_event) {
30eb77f4 2977 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
77b1bad4 2978 /* flush read cache over gart */
e32eb50d
CK		 2979 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2980 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2981 PACKET3_VC_ACTION_ENA |
2982 PACKET3_SH_ACTION_ENA);
2983 radeon_ring_write(ring, 0xFFFFFFFF);
2984 radeon_ring_write(ring, 0);
2985 radeon_ring_write(ring, 10); /* poll interval */
d0f8a854 2986 /* EVENT_WRITE_EOP - flush caches, send int */
e32eb50d
CK		 2987 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2988 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
2989 radeon_ring_write(ring, addr & 0xffffffff);
2990 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2991 radeon_ring_write(ring, fence->seq);
2992 radeon_ring_write(ring, 0);
d0f8a854 2993 } else {
77b1bad4 2994 /* flush read cache over gart */
e32eb50d
CK		 2995 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2996 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2997 PACKET3_VC_ACTION_ENA |
2998 PACKET3_SH_ACTION_ENA);
2999 radeon_ring_write(ring, 0xFFFFFFFF);
3000 radeon_ring_write(ring, 0);
3001 radeon_ring_write(ring, 10); /* poll interval */
3002 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
3003 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
d0f8a854 3004 /* wait for 3D idle clean */
e32eb50d
CK		 3005 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3006 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3007 radeon_ring_write(ring, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
d0f8a854 3008 /* Emit fence sequence & fire IRQ */
e32eb50d
CK		 3009 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3010 radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
3011 radeon_ring_write(ring, fence->seq);
d0f8a854 3012 /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
e32eb50d
CK		 3013 radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
3014 radeon_ring_write(ring, RB_INT_STAT);
d0f8a854 3015 }
3ce0a23d
JG		 3016}
3017
f2ba57b5
CK		 3018void r600_uvd_fence_emit(struct radeon_device *rdev,
3019 struct radeon_fence *fence)
3020{
3021 struct radeon_ring *ring = &rdev->ring[fence->ring];
3022 uint32_t addr = rdev->fence_drv[fence->ring].gpu_addr;
3023
3024 radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0));
3025 radeon_ring_write(ring, fence->seq);
3026 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0));
3027 radeon_ring_write(ring, addr & 0xffffffff);
3028 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0));
3029 radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
3030 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0));
3031 radeon_ring_write(ring, 0);
3032
3033 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0));
3034 radeon_ring_write(ring, 0);
3035 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0));
3036 radeon_ring_write(ring, 0);
3037 radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0));
3038 radeon_ring_write(ring, 2);
3039 return;
3040}
3041
15d3332f 3042void r600_semaphore_ring_emit(struct radeon_device *rdev,
e32eb50d 3043 struct radeon_ring *ring,
15d3332f 3044 struct radeon_semaphore *semaphore,
7b1f2485 3045 bool emit_wait)
15d3332f
CK		 3046{
3047 uint64_t addr = semaphore->gpu_addr;
3048 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3049
0be70439
CK		 3050 if (rdev->family < CHIP_CAYMAN)
3051 sel |= PACKET3_SEM_WAIT_ON_SIGNAL;
3052
e32eb50d
CK		 3053 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3054 radeon_ring_write(ring, addr & 0xffffffff);
3055 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | sel);
15d3332f
CK		 3056}
3057
4d75658b
AD		 3058/*
3059 * DMA fences/semaphores
3060 */
3061
3062/**
3063 * r600_dma_fence_ring_emit - emit a fence on the DMA ring
3064 *
3065 * @rdev: radeon_device pointer
3066 * @fence: radeon fence object
3067 *
3068 * Add a DMA fence packet to the ring to write
3069 * the fence seq number and DMA trap packet to generate
3070 * an interrupt if needed (r6xx-r7xx).
3071 */
3072void r600_dma_fence_ring_emit(struct radeon_device *rdev,
3073 struct radeon_fence *fence)
3074{
3075 struct radeon_ring *ring = &rdev->ring[fence->ring];
3076 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
86a1881d 3077
4d75658b
AD		 3078 /* write the fence */
3079 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0));
3080 radeon_ring_write(ring, addr & 0xfffffffc);
3081 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
86a1881d 3082 radeon_ring_write(ring, lower_32_bits(fence->seq));
4d75658b
AD		 3083 /* generate an interrupt */
3084 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0));
3085}
3086
3087/**
3088 * r600_dma_semaphore_ring_emit - emit a semaphore on the dma ring
3089 *
3090 * @rdev: radeon_device pointer
3091 * @ring: radeon_ring structure holding ring information
3092 * @semaphore: radeon semaphore object
3093 * @emit_wait: wait or signal semaphore
3094 *
3095 * Add a DMA semaphore packet to the ring wait on or signal
3096 * other rings (r6xx-SI).
3097 */
3098void r600_dma_semaphore_ring_emit(struct radeon_device *rdev,
3099 struct radeon_ring *ring,
3100 struct radeon_semaphore *semaphore,
3101 bool emit_wait)
3102{
3103 u64 addr = semaphore->gpu_addr;
3104 u32 s = emit_wait ? 0 : 1;
3105
3106 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SEMAPHORE, 0, s, 0));
3107 radeon_ring_write(ring, addr & 0xfffffffc);
3108 radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
3109}
3110
f2ba57b5
CK		 3111void r600_uvd_semaphore_emit(struct radeon_device *rdev,
3112 struct radeon_ring *ring,
3113 struct radeon_semaphore *semaphore,
3114 bool emit_wait)
3115{
3116 uint64_t addr = semaphore->gpu_addr;
3117
3118 radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0));
3119 radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF);
3120
3121 radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0));
3122 radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF);
3123
3124 radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0));
3125 radeon_ring_write(ring, emit_wait ? 1 : 0);
3126}
3127
3ce0a23d 3128int r600_copy_blit(struct radeon_device *rdev,
003cefe0
AD		 3129 uint64_t src_offset,
3130 uint64_t dst_offset,
3131 unsigned num_gpu_pages,
876dc9f3 3132 struct radeon_fence **fence)
3ce0a23d 3133{
220907d9 3134 struct radeon_semaphore *sem = NULL;
f237750f 3135 struct radeon_sa_bo *vb = NULL;
ff82f052
JG		 3136 int r;
3137
220907d9 3138 r = r600_blit_prepare_copy(rdev, num_gpu_pages, fence, &vb, &sem);
ff82f052 3139 if (r) {
ff82f052
JG		 3140 return r;
3141 }
f237750f 3142 r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages, vb);
220907d9 3143 r600_blit_done_copy(rdev, fence, vb, sem);
3ce0a23d
JG		 3144 return 0;
3145}
3146
072b5acc
AD		 3147/**
3148 * r600_copy_cpdma - copy pages using the CP DMA engine
3149 *
3150 * @rdev: radeon_device pointer
3151 * @src_offset: src GPU address
3152 * @dst_offset: dst GPU address
3153 * @num_gpu_pages: number of GPU pages to xfer
3154 * @fence: radeon fence object
3155 *
3156 * Copy GPU paging using the CP DMA engine (r6xx+).
3157 * Used by the radeon ttm implementation to move pages if
3158 * registered as the asic copy callback.
3159 */
3160int r600_copy_cpdma(struct radeon_device *rdev,
3161 uint64_t src_offset, uint64_t dst_offset,
3162 unsigned num_gpu_pages,
3163 struct radeon_fence **fence)
3164{
3165 struct radeon_semaphore *sem = NULL;
3166 int ring_index = rdev->asic->copy.blit_ring_index;
3167 struct radeon_ring *ring = &rdev->ring[ring_index];
3168 u32 size_in_bytes, cur_size_in_bytes, tmp;
3169 int i, num_loops;
3170 int r = 0;
3171
3172 r = radeon_semaphore_create(rdev, &sem);
3173 if (r) {
3174 DRM_ERROR("radeon: moving bo (%d).\n", r);
3175 return r;
3176 }
3177
3178 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3179 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3180 r = radeon_ring_lock(rdev, ring, num_loops * 6 + 21);
3181 if (r) {
3182 DRM_ERROR("radeon: moving bo (%d).\n", r);
3183 radeon_semaphore_free(rdev, &sem, NULL);
3184 return r;
3185 }
3186
3187 if (radeon_fence_need_sync(*fence, ring->idx)) {
3188 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3189 ring->idx);
3190 radeon_fence_note_sync(*fence, ring->idx);
3191 } else {
3192 radeon_semaphore_free(rdev, &sem, NULL);
3193 }
3194
3195 for (i = 0; i < num_loops; i++) {
3196 cur_size_in_bytes = size_in_bytes;
3197 if (cur_size_in_bytes > 0x1fffff)
3198 cur_size_in_bytes = 0x1fffff;
3199 size_in_bytes -= cur_size_in_bytes;
3200 tmp = upper_32_bits(src_offset) & 0xff;
3201 if (size_in_bytes == 0)
3202 tmp |= PACKET3_CP_DMA_CP_SYNC;
3203 radeon_ring_write(ring, PACKET3(PACKET3_CP_DMA, 4));
3204 radeon_ring_write(ring, src_offset & 0xffffffff);
3205 radeon_ring_write(ring, tmp);
3206 radeon_ring_write(ring, dst_offset & 0xffffffff);
3207 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
3208 radeon_ring_write(ring, cur_size_in_bytes);
3209 src_offset += cur_size_in_bytes;
3210 dst_offset += cur_size_in_bytes;
3211 }
3212 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3213 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3214 radeon_ring_write(ring, WAIT_CP_DMA_IDLE_bit);
3215
3216 r = radeon_fence_emit(rdev, fence, ring->idx);
3217 if (r) {
3218 radeon_ring_unlock_undo(rdev, ring);
3219 return r;
3220 }
3221
3222 radeon_ring_unlock_commit(rdev, ring);
3223 radeon_semaphore_free(rdev, &sem, *fence);
3224
3225 return r;
3226}
3227
4d75658b
AD		 3228/**
3229 * r600_copy_dma - copy pages using the DMA engine
3230 *
3231 * @rdev: radeon_device pointer
3232 * @src_offset: src GPU address
3233 * @dst_offset: dst GPU address
3234 * @num_gpu_pages: number of GPU pages to xfer
3235 * @fence: radeon fence object
3236 *
43fb7787 3237 * Copy GPU paging using the DMA engine (r6xx).
4d75658b
AD		 3238 * Used by the radeon ttm implementation to move pages if
3239 * registered as the asic copy callback.
3240 */
3241int r600_copy_dma(struct radeon_device *rdev,
3242 uint64_t src_offset, uint64_t dst_offset,
3243 unsigned num_gpu_pages,
3244 struct radeon_fence **fence)
3245{
3246 struct radeon_semaphore *sem = NULL;
3247 int ring_index = rdev->asic->copy.dma_ring_index;
3248 struct radeon_ring *ring = &rdev->ring[ring_index];
3249 u32 size_in_dw, cur_size_in_dw;
3250 int i, num_loops;
3251 int r = 0;
3252
3253 r = radeon_semaphore_create(rdev, &sem);
3254 if (r) {
3255 DRM_ERROR("radeon: moving bo (%d).\n", r);
3256 return r;
3257 }
3258
3259 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
43fb7787
AD		 3260 num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE);
3261 r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8);
4d75658b
AD		 3262 if (r) {
3263 DRM_ERROR("radeon: moving bo (%d).\n", r);
3264 radeon_semaphore_free(rdev, &sem, NULL);
3265 return r;
3266 }
3267
3268 if (radeon_fence_need_sync(*fence, ring->idx)) {
3269 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3270 ring->idx);
3271 radeon_fence_note_sync(*fence, ring->idx);
3272 } else {
3273 radeon_semaphore_free(rdev, &sem, NULL);
3274 }
3275
3276 for (i = 0; i < num_loops; i++) {
3277 cur_size_in_dw = size_in_dw;
909d9eb6
AD		 3278 if (cur_size_in_dw > 0xFFFE)
3279 cur_size_in_dw = 0xFFFE;
4d75658b
AD		 3280 size_in_dw -= cur_size_in_dw;
3281 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw));
3282 radeon_ring_write(ring, dst_offset & 0xfffffffc);
3283 radeon_ring_write(ring, src_offset & 0xfffffffc);
43fb7787
AD		 3284 radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) |
3285 (upper_32_bits(src_offset) & 0xff)));
4d75658b
AD		 3286 src_offset += cur_size_in_dw * 4;
3287 dst_offset += cur_size_in_dw * 4;
3288 }
3289
3290 r = radeon_fence_emit(rdev, fence, ring->idx);
3291 if (r) {
3292 radeon_ring_unlock_undo(rdev, ring);
3293 return r;
3294 }
3295
3296 radeon_ring_unlock_commit(rdev, ring);
3297 radeon_semaphore_free(rdev, &sem, *fence);
3298
3299 return r;
3300}
3301
3ce0a23d
JG		 3302int r600_set_surface_reg(struct radeon_device *rdev, int reg,
3303 uint32_t tiling_flags, uint32_t pitch,
3304 uint32_t offset, uint32_t obj_size)
3305{
3306 /* FIXME: implement */
3307 return 0;
3308}
3309
3310void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
3311{
3312 /* FIXME: implement */
3313}
3314
1109ca09 3315static int r600_startup(struct radeon_device *rdev)
3ce0a23d 3316{
4d75658b 3317 struct radeon_ring *ring;
3ce0a23d
JG		 3318 int r;
3319
9e46a48d
AD		 3320 /* enable pcie gen2 link */
3321 r600_pcie_gen2_enable(rdev);
3322
779720a3
AD		 3323 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3324 r = r600_init_microcode(rdev);
3325 if (r) {
3326 DRM_ERROR("Failed to load firmware!\n");
3327 return r;
3328 }
3329 }
3330
16cdf04d
AD		 3331 r = r600_vram_scratch_init(rdev);
3332 if (r)
3333 return r;
3334
a3c1945a 3335 r600_mc_program(rdev);
1a029b76
JG		 3336 if (rdev->flags & RADEON_IS_AGP) {
3337 r600_agp_enable(rdev);
3338 } else {
3339 r = r600_pcie_gart_enable(rdev);
3340 if (r)
3341 return r;
3342 }
3ce0a23d 3343 r600_gpu_init(rdev);
c38c7b64
JG		 3344 r = r600_blit_init(rdev);
3345 if (r) {
3346 r600_blit_fini(rdev);
27cd7769 3347 rdev->asic->copy.copy = NULL;
c38c7b64
JG		 3348 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3349 }
b70d6bb3 3350
724c80e1
AD		 3351 /* allocate wb buffer */
3352 r = radeon_wb_init(rdev);
3353 if (r)
3354 return r;
3355
30eb77f4
JG		 3356 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
3357 if (r) {
3358 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3359 return r;
3360 }
3361
4d75658b
AD		 3362 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
3363 if (r) {
3364 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
3365 return r;
3366 }
3367
d8f60cfc 3368 /* Enable IRQ */
e49f3959
AH		 3369 if (!rdev->irq.installed) {
3370 r = radeon_irq_kms_init(rdev);
3371 if (r)
3372 return r;
3373 }
3374
d8f60cfc
AD		 3375 r = r600_irq_init(rdev);
3376 if (r) {
3377 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3378 radeon_irq_kms_fini(rdev);
3379 return r;
3380 }
3381 r600_irq_set(rdev);
3382
4d75658b 3383 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 3384 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a
AD		 3385 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
3386 0, 0xfffff, RADEON_CP_PACKET2);
4d75658b
AD		 3387 if (r)
3388 return r;
5596a9db 3389
4d75658b
AD		 3390 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
3391 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
3392 DMA_RB_RPTR, DMA_RB_WPTR,
3393 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3ce0a23d
JG		 3394 if (r)
3395 return r;
4d75658b 3396
3ce0a23d
JG		 3397 r = r600_cp_load_microcode(rdev);
3398 if (r)
3399 return r;
3400 r = r600_cp_resume(rdev);
3401 if (r)
3402 return r;
724c80e1 3403
4d75658b
AD		 3404 r = r600_dma_resume(rdev);
3405 if (r)
3406 return r;
3407
2898c348
CK		 3408 r = radeon_ib_pool_init(rdev);
3409 if (r) {
3410 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 3411 return r;
2898c348 3412 }
b15ba512 3413
d4e30ef0
AD		 3414 r = r600_audio_init(rdev);
3415 if (r) {
3416 DRM_ERROR("radeon: audio init failed\n");
3417 return r;
3418 }
3419
3ce0a23d
JG		 3420 return 0;
3421}
3422
28d52043
DA		 3423void r600_vga_set_state(struct radeon_device *rdev, bool state)
3424{
3425 uint32_t temp;
3426
3427 temp = RREG32(CONFIG_CNTL);
3428 if (state == false) {
3429 temp &= ~(1<<0);
3430 temp |= (1<<1);
3431 } else {
3432 temp &= ~(1<<1);
3433 }
3434 WREG32(CONFIG_CNTL, temp);
3435}
3436
fc30b8ef
DA		 3437int r600_resume(struct radeon_device *rdev)
3438{
3439 int r;
3440
1a029b76
JG		 3441 /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
3442 * posting will perform necessary task to bring back GPU into good
3443 * shape.
3444 */
fc30b8ef 3445 /* post card */
e7d40b9a 3446 atom_asic_init(rdev->mode_info.atom_context);
fc30b8ef 3447
b15ba512 3448 rdev->accel_working = true;
fc30b8ef
DA		 3449 r = r600_startup(rdev);
3450 if (r) {
3451 DRM_ERROR("r600 startup failed on resume\n");
6b7746e8 3452 rdev->accel_working = false;
fc30b8ef
DA		 3453 return r;
3454 }
3455
fc30b8ef
DA		 3456 return r;
3457}
3458
3ce0a23d
JG		 3459int r600_suspend(struct radeon_device *rdev)
3460{
38fd2c6f 3461 r600_audio_fini(rdev);
3ce0a23d 3462 r600_cp_stop(rdev);
4d75658b 3463 r600_dma_stop(rdev);
0c45249f 3464 r600_irq_suspend(rdev);
724c80e1 3465 radeon_wb_disable(rdev);
4aac0473 3466 r600_pcie_gart_disable(rdev);
6ddddfe7 3467
3ce0a23d
JG		 3468 return 0;
3469}
3470
3471/* Plan is to move initialization in that function and use
3472 * helper function so that radeon_device_init pretty much
3473 * do nothing more than calling asic specific function. This
3474 * should also allow to remove a bunch of callback function
3475 * like vram_info.
3476 */
3477int r600_init(struct radeon_device *rdev)
771fe6b9 3478{
3ce0a23d 3479 int r;
771fe6b9 3480
3ce0a23d
JG		 3481 if (r600_debugfs_mc_info_init(rdev)) {
3482 DRM_ERROR("Failed to register debugfs file for mc !\n");
3483 }
3ce0a23d
JG		 3484 /* Read BIOS */
3485 if (!radeon_get_bios(rdev)) {
3486 if (ASIC_IS_AVIVO(rdev))
3487 return -EINVAL;
3488 }
3489 /* Must be an ATOMBIOS */
e7d40b9a
JG		 3490 if (!rdev->is_atom_bios) {
3491 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
3ce0a23d 3492 return -EINVAL;
e7d40b9a 3493 }
3ce0a23d
JG		 3494 r = radeon_atombios_init(rdev);
3495 if (r)
3496 return r;
3497 /* Post card if necessary */
fd909c37 3498 if (!radeon_card_posted(rdev)) {
72542d77
DA		 3499 if (!rdev->bios) {
3500 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3501 return -EINVAL;
3502 }
3ce0a23d
JG		 3503 DRM_INFO("GPU not posted. posting now...\n");
3504 atom_asic_init(rdev->mode_info.atom_context);
3505 }
3506 /* Initialize scratch registers */
3507 r600_scratch_init(rdev);
3508 /* Initialize surface registers */
3509 radeon_surface_init(rdev);
7433874e 3510 /* Initialize clocks */
5e6dde7e 3511 radeon_get_clock_info(rdev->ddev);
3ce0a23d 3512 /* Fence driver */
30eb77f4 3513 r = radeon_fence_driver_init(rdev);
3ce0a23d
JG		 3514 if (r)
3515 return r;
700a0cc0
JG		 3516 if (rdev->flags & RADEON_IS_AGP) {
3517 r = radeon_agp_init(rdev);
3518 if (r)
3519 radeon_agp_disable(rdev);
3520 }
3ce0a23d 3521 r = r600_mc_init(rdev);
b574f251 3522 if (r)
3ce0a23d 3523 return r;
3ce0a23d 3524 /* Memory manager */
4c788679 3525 r = radeon_bo_init(rdev);
3ce0a23d
JG		 3526 if (r)
3527 return r;
d8f60cfc 3528
e32eb50d
CK		 3529 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
3530 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
3ce0a23d 3531
4d75658b
AD		 3532 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
3533 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
3534
d8f60cfc
AD		 3535 rdev->ih.ring_obj = NULL;
3536 r600_ih_ring_init(rdev, 64 * 1024);
3ce0a23d 3537
4aac0473
JG		 3538 r = r600_pcie_gart_init(rdev);
3539 if (r)
3540 return r;
3541
779720a3 3542 rdev->accel_working = true;
fc30b8ef 3543 r = r600_startup(rdev);
3ce0a23d 3544 if (r) {
655efd3d
JG		 3545 dev_err(rdev->dev, "disabling GPU acceleration\n");
3546 r600_cp_fini(rdev);
4d75658b 3547 r600_dma_fini(rdev);
655efd3d 3548 r600_irq_fini(rdev);
724c80e1 3549 radeon_wb_fini(rdev);
2898c348 3550 radeon_ib_pool_fini(rdev);
655efd3d 3551 radeon_irq_kms_fini(rdev);
75c81298 3552 r600_pcie_gart_fini(rdev);
733289c2 3553 rdev->accel_working = false;
3ce0a23d 3554 }
dafc3bd5 3555
3ce0a23d
JG		 3556 return 0;
3557}
3558
3559void r600_fini(struct radeon_device *rdev)
3560{
dafc3bd5 3561 r600_audio_fini(rdev);
3ce0a23d 3562 r600_blit_fini(rdev);
655efd3d 3563 r600_cp_fini(rdev);
4d75658b 3564 r600_dma_fini(rdev);
d8f60cfc 3565 r600_irq_fini(rdev);
724c80e1 3566 radeon_wb_fini(rdev);
2898c348 3567 radeon_ib_pool_fini(rdev);
d8f60cfc 3568 radeon_irq_kms_fini(rdev);
4aac0473 3569 r600_pcie_gart_fini(rdev);
16cdf04d 3570 r600_vram_scratch_fini(rdev);
655efd3d 3571 radeon_agp_fini(rdev);
3ce0a23d
JG		 3572 radeon_gem_fini(rdev);
3573 radeon_fence_driver_fini(rdev);
4c788679 3574 radeon_bo_fini(rdev);
e7d40b9a 3575 radeon_atombios_fini(rdev);
3ce0a23d
JG		 3576 kfree(rdev->bios);
3577 rdev->bios = NULL;
3ce0a23d
JG		 3578}
3579
3580
3581/*
3582 * CS stuff
3583 */
3584void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3585{
876dc9f3 3586 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 3587 u32 next_rptr;
7b1f2485 3588
45df6803 3589 if (ring->rptr_save_reg) {
89d35807 3590 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 3591 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3592 radeon_ring_write(ring, ((ring->rptr_save_reg -
3593 PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
3594 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3595 } else if (rdev->wb.enabled) {
3596 next_rptr = ring->wptr + 5 + 4;
3597 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3598 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3599 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3600 radeon_ring_write(ring, next_rptr);
3601 radeon_ring_write(ring, 0);
45df6803
CK		 3602 }
3603
e32eb50d
CK		 3604 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3605 radeon_ring_write(ring,
4eace7fd
CC		 3606#ifdef __BIG_ENDIAN
3607 (2 << 0) |
3608#endif
3609 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 3610 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3611 radeon_ring_write(ring, ib->length_dw);
3ce0a23d
JG		 3612}
3613
f2ba57b5
CK		 3614void r600_uvd_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3615{
3616 struct radeon_ring *ring = &rdev->ring[ib->ring];
3617
3618 radeon_ring_write(ring, PACKET0(UVD_RBC_IB_BASE, 0));
3619 radeon_ring_write(ring, ib->gpu_addr);
3620 radeon_ring_write(ring, PACKET0(UVD_RBC_IB_SIZE, 0));
3621 radeon_ring_write(ring, ib->length_dw);
3622}
3623
f712812e 3624int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3ce0a23d 3625{
f2e39221 3626 struct radeon_ib ib;
3ce0a23d
JG		 3627 uint32_t scratch;
3628 uint32_t tmp = 0;
3629 unsigned i;
3630 int r;
3631
3632 r = radeon_scratch_get(rdev, &scratch);
3633 if (r) {
3634 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3635 return r;
3636 }
3637 WREG32(scratch, 0xCAFEDEAD);
4bf3dd92 3638 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3ce0a23d
JG		 3639 if (r) {
3640 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
af026c5b 3641 goto free_scratch;
3ce0a23d 3642 }
f2e39221
JG		 3643 ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
3644 ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3645 ib.ptr[2] = 0xDEADBEEF;
3646 ib.length_dw = 3;
4ef72566 3647 r = radeon_ib_schedule(rdev, &ib, NULL);
3ce0a23d 3648 if (r) {
3ce0a23d 3649 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
af026c5b 3650 goto free_ib;
3ce0a23d 3651 }
f2e39221 3652 r = radeon_fence_wait(ib.fence, false);
3ce0a23d
JG		 3653 if (r) {
3654 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
af026c5b 3655 goto free_ib;
3ce0a23d
JG		 3656 }
3657 for (i = 0; i < rdev->usec_timeout; i++) {
3658 tmp = RREG32(scratch);
3659 if (tmp == 0xDEADBEEF)
3660 break;
3661 DRM_UDELAY(1);
3662 }
3663 if (i < rdev->usec_timeout) {
f2e39221 3664 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3ce0a23d 3665 } else {
4417d7f6 3666 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3ce0a23d
JG		 3667 scratch, tmp);
3668 r = -EINVAL;
3669 }
af026c5b 3670free_ib:
3ce0a23d 3671 radeon_ib_free(rdev, &ib);
af026c5b
MD		 3672free_scratch:
3673 radeon_scratch_free(rdev, scratch);
771fe6b9
JG		 3674 return r;
3675}
3676
4d75658b
AD		 3677/**
3678 * r600_dma_ib_test - test an IB on the DMA engine
3679 *
3680 * @rdev: radeon_device pointer
3681 * @ring: radeon_ring structure holding ring information
3682 *
3683 * Test a simple IB in the DMA ring (r6xx-SI).
3684 * Returns 0 on success, error on failure.
3685 */
3686int r600_dma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3687{
3688 struct radeon_ib ib;
3689 unsigned i;
3690 int r;
3691 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
3692 u32 tmp = 0;
3693
3694 if (!ptr) {
3695 DRM_ERROR("invalid vram scratch pointer\n");
3696 return -EINVAL;
3697 }
3698
3699 tmp = 0xCAFEDEAD;
3700 writel(tmp, ptr);
3701
3702 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3703 if (r) {
3704 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3705 return r;
3706 }
3707
3708 ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1);
3709 ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
3710 ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff;
3711 ib.ptr[3] = 0xDEADBEEF;
3712 ib.length_dw = 4;
3713
3714 r = radeon_ib_schedule(rdev, &ib, NULL);
3715 if (r) {
3716 radeon_ib_free(rdev, &ib);
3717 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3718 return r;
3719 }
3720 r = radeon_fence_wait(ib.fence, false);
3721 if (r) {
3722 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3723 return r;
3724 }
3725 for (i = 0; i < rdev->usec_timeout; i++) {
3726 tmp = readl(ptr);
3727 if (tmp == 0xDEADBEEF)
3728 break;
3729 DRM_UDELAY(1);
3730 }
3731 if (i < rdev->usec_timeout) {
3732 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3733 } else {
3734 DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
3735 r = -EINVAL;
3736 }
3737 radeon_ib_free(rdev, &ib);
3738 return r;
3739}
3740
f2ba57b5
CK		 3741int r600_uvd_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3742{
b05e9e4c 3743 struct radeon_fence *fence = NULL;
f2ba57b5
CK		 3744 int r;
3745
b05e9e4c
CK		 3746 r = radeon_set_uvd_clocks(rdev, 53300, 40000);
3747 if (r) {
3748 DRM_ERROR("radeon: failed to raise UVD clocks (%d).\n", r);
3749 return r;
3750 }
3751
f2ba57b5
CK		 3752 r = radeon_uvd_get_create_msg(rdev, ring->idx, 1, NULL);
3753 if (r) {
3754 DRM_ERROR("radeon: failed to get create msg (%d).\n", r);
b05e9e4c 3755 goto error;
f2ba57b5
CK		 3756 }
3757
3758 r = radeon_uvd_get_destroy_msg(rdev, ring->idx, 1, &fence);
3759 if (r) {
3760 DRM_ERROR("radeon: failed to get destroy ib (%d).\n", r);
b05e9e4c 3761 goto error;
f2ba57b5
CK		 3762 }
3763
3764 r = radeon_fence_wait(fence, false);
3765 if (r) {
3766 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
b05e9e4c 3767 goto error;
f2ba57b5
CK		 3768 }
3769 DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
b05e9e4c 3770error:
f2ba57b5 3771 radeon_fence_unref(&fence);
b05e9e4c 3772 radeon_set_uvd_clocks(rdev, 0, 0);
f2ba57b5
CK		 3773 return r;
3774}
3775
4d75658b
AD		 3776/**
3777 * r600_dma_ring_ib_execute - Schedule an IB on the DMA engine
3778 *
3779 * @rdev: radeon_device pointer
3780 * @ib: IB object to schedule
3781 *
3782 * Schedule an IB in the DMA ring (r6xx-r7xx).
3783 */
3784void r600_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3785{
3786 struct radeon_ring *ring = &rdev->ring[ib->ring];
3787
3788 if (rdev->wb.enabled) {
3789 u32 next_rptr = ring->wptr + 4;
3790 while ((next_rptr & 7) != 5)
3791 next_rptr++;
3792 next_rptr += 3;
3793 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
3794 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3795 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
3796 radeon_ring_write(ring, next_rptr);
3797 }
3798
3799 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
3800 * Pad as necessary with NOPs.
3801 */
3802 while ((ring->wptr & 7) != 5)
3803 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3804 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0));
3805 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
3806 radeon_ring_write(ring, (ib->length_dw << 16) | (upper_32_bits(ib->gpu_addr) & 0xFF));
3807
3808}
3809
d8f60cfc
AD		 3810/*
3811 * Interrupts
3812 *
3813 * Interrupts use a ring buffer on r6xx/r7xx hardware. It works pretty
3814 * the same as the CP ring buffer, but in reverse. Rather than the CPU
3815 * writing to the ring and the GPU consuming, the GPU writes to the ring
3816 * and host consumes. As the host irq handler processes interrupts, it
3817 * increments the rptr. When the rptr catches up with the wptr, all the
3818 * current interrupts have been processed.
3819 */
3820
3821void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
3822{
3823 u32 rb_bufsz;
3824
3825 /* Align ring size */
3826 rb_bufsz = drm_order(ring_size / 4);
3827 ring_size = (1 << rb_bufsz) * 4;
3828 rdev->ih.ring_size = ring_size;
0c45249f
JG		 3829 rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
3830 rdev->ih.rptr = 0;
d8f60cfc
AD		 3831}
3832
25a857fb 3833int r600_ih_ring_alloc(struct radeon_device *rdev)
d8f60cfc
AD		 3834{
3835 int r;
3836
d8f60cfc
AD		 3837 /* Allocate ring buffer */
3838 if (rdev->ih.ring_obj == NULL) {
441921d5 3839 r = radeon_bo_create(rdev, rdev->ih.ring_size,
268b2510 3840 PAGE_SIZE, true,
4c788679 3841 RADEON_GEM_DOMAIN_GTT,
40f5cf99 3842 NULL, &rdev->ih.ring_obj);
d8f60cfc
AD		 3843 if (r) {
3844 DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
3845 return r;
3846 }
4c788679
JG		 3847 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3848 if (unlikely(r != 0))
3849 return r;
3850 r = radeon_bo_pin(rdev->ih.ring_obj,
3851 RADEON_GEM_DOMAIN_GTT,
3852 &rdev->ih.gpu_addr);
d8f60cfc 3853 if (r) {
4c788679 3854 radeon_bo_unreserve(rdev->ih.ring_obj);
d8f60cfc
AD		 3855 DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
3856 return r;
3857 }
4c788679
JG		 3858 r = radeon_bo_kmap(rdev->ih.ring_obj,
3859 (void **)&rdev->ih.ring);
3860 radeon_bo_unreserve(rdev->ih.ring_obj);
d8f60cfc
AD		 3861 if (r) {
3862 DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
3863 return r;
3864 }
3865 }
d8f60cfc
AD		 3866 return 0;
3867}
3868
25a857fb 3869void r600_ih_ring_fini(struct radeon_device *rdev)
d8f60cfc 3870{
4c788679 3871 int r;
d8f60cfc 3872 if (rdev->ih.ring_obj) {
4c788679
JG		 3873 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3874 if (likely(r == 0)) {
3875 radeon_bo_kunmap(rdev->ih.ring_obj);
3876 radeon_bo_unpin(rdev->ih.ring_obj);
3877 radeon_bo_unreserve(rdev->ih.ring_obj);
3878 }
3879 radeon_bo_unref(&rdev->ih.ring_obj);
d8f60cfc
AD		 3880 rdev->ih.ring = NULL;
3881 rdev->ih.ring_obj = NULL;
3882 }
3883}
3884
45f9a39b 3885void r600_rlc_stop(struct radeon_device *rdev)
d8f60cfc
AD		 3886{
3887
45f9a39b
AD		 3888 if ((rdev->family >= CHIP_RV770) &&
3889 (rdev->family <= CHIP_RV740)) {
d8f60cfc
AD		 3890 /* r7xx asics need to soft reset RLC before halting */
3891 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
3892 RREG32(SRBM_SOFT_RESET);
4de833c3 3893 mdelay(15);
d8f60cfc
AD		 3894 WREG32(SRBM_SOFT_RESET, 0);
3895 RREG32(SRBM_SOFT_RESET);
3896 }
3897
3898 WREG32(RLC_CNTL, 0);
3899}
3900
3901static void r600_rlc_start(struct radeon_device *rdev)
3902{
3903 WREG32(RLC_CNTL, RLC_ENABLE);
3904}
3905
2948f5e6 3906static int r600_rlc_resume(struct radeon_device *rdev)
d8f60cfc
AD		 3907{
3908 u32 i;
3909 const __be32 *fw_data;
3910
3911 if (!rdev->rlc_fw)
3912 return -EINVAL;
3913
3914 r600_rlc_stop(rdev);
3915
d8f60cfc 3916 WREG32(RLC_HB_CNTL, 0);
c420c745 3917
2948f5e6
AD		 3918 WREG32(RLC_HB_BASE, 0);
3919 WREG32(RLC_HB_RPTR, 0);
3920 WREG32(RLC_HB_WPTR, 0);
3921 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
3922 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
d8f60cfc
AD		 3923 WREG32(RLC_MC_CNTL, 0);
3924 WREG32(RLC_UCODE_CNTL, 0);
3925
3926 fw_data = (const __be32 *)rdev->rlc_fw->data;
2948f5e6 3927 if (rdev->family >= CHIP_RV770) {
d8f60cfc
AD		 3928 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
3929 WREG32(RLC_UCODE_ADDR, i);
3930 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3931 }
3932 } else {
138e4e16 3933 for (i = 0; i < R600_RLC_UCODE_SIZE; i++) {
d8f60cfc
AD		 3934 WREG32(RLC_UCODE_ADDR, i);
3935 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3936 }
3937 }
3938 WREG32(RLC_UCODE_ADDR, 0);
3939
3940 r600_rlc_start(rdev);
3941
3942 return 0;
3943}
3944
3945static void r600_enable_interrupts(struct radeon_device *rdev)
3946{
3947 u32 ih_cntl = RREG32(IH_CNTL);
3948 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3949
3950 ih_cntl |= ENABLE_INTR;
3951 ih_rb_cntl |= IH_RB_ENABLE;
3952 WREG32(IH_CNTL, ih_cntl);
3953 WREG32(IH_RB_CNTL, ih_rb_cntl);
3954 rdev->ih.enabled = true;
3955}
3956
45f9a39b 3957void r600_disable_interrupts(struct radeon_device *rdev)
d8f60cfc
AD		 3958{
3959 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3960 u32 ih_cntl = RREG32(IH_CNTL);
3961
3962 ih_rb_cntl &= ~IH_RB_ENABLE;
3963 ih_cntl &= ~ENABLE_INTR;
3964 WREG32(IH_RB_CNTL, ih_rb_cntl);
3965 WREG32(IH_CNTL, ih_cntl);
3966 /* set rptr, wptr to 0 */
3967 WREG32(IH_RB_RPTR, 0);
3968 WREG32(IH_RB_WPTR, 0);
3969 rdev->ih.enabled = false;
d8f60cfc
AD		 3970 rdev->ih.rptr = 0;
3971}
3972
e0df1ac5
AD		 3973static void r600_disable_interrupt_state(struct radeon_device *rdev)
3974{
3975 u32 tmp;
3976
3555e53b 3977 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4d75658b
AD		 3978 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3979 WREG32(DMA_CNTL, tmp);
e0df1ac5
AD		 3980 WREG32(GRBM_INT_CNTL, 0);
3981 WREG32(DxMODE_INT_MASK, 0);
6f34be50
AD		 3982 WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
3983 WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
e0df1ac5
AD		 3984 if (ASIC_IS_DCE3(rdev)) {
3985 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
3986 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
3987 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3988 WREG32(DC_HPD1_INT_CONTROL, tmp);
3989 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3990 WREG32(DC_HPD2_INT_CONTROL, tmp);
3991 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3992 WREG32(DC_HPD3_INT_CONTROL, tmp);
3993 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3994 WREG32(DC_HPD4_INT_CONTROL, tmp);
3995 if (ASIC_IS_DCE32(rdev)) {
3996 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5898b1f3 3997 WREG32(DC_HPD5_INT_CONTROL, tmp);
e0df1ac5 3998 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5898b1f3 3999 WREG32(DC_HPD6_INT_CONTROL, tmp);
c6543a6e
RM		 4000 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4001 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
4002 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4003 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
f122c610
AD		 4004 } else {
4005 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4006 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
4007 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4008 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
e0df1ac5
AD		 4009 }
4010 } else {
4011 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4012 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4013 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
5898b1f3 4014 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
e0df1ac5 4015 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
5898b1f3 4016 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
e0df1ac5 4017 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
5898b1f3 4018 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
f122c610
AD		 4019 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4020 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
4021 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4022 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
e0df1ac5
AD		 4023 }
4024}
4025
d8f60cfc
AD		 4026int r600_irq_init(struct radeon_device *rdev)
4027{
4028 int ret = 0;
4029 int rb_bufsz;
4030 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
4031
4032 /* allocate ring */
0c45249f 4033 ret = r600_ih_ring_alloc(rdev);
d8f60cfc
AD		 4034 if (ret)
4035 return ret;
4036
4037 /* disable irqs */
4038 r600_disable_interrupts(rdev);
4039
4040 /* init rlc */
2948f5e6
AD		 4041 if (rdev->family >= CHIP_CEDAR)
4042 ret = evergreen_rlc_resume(rdev);
4043 else
4044 ret = r600_rlc_resume(rdev);
d8f60cfc
AD		 4045 if (ret) {
4046 r600_ih_ring_fini(rdev);
4047 return ret;
4048 }
4049
4050 /* setup interrupt control */
4051 /* set dummy read address to ring address */
4052 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
4053 interrupt_cntl = RREG32(INTERRUPT_CNTL);
4054 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
4055 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
4056 */
4057 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
4058 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
4059 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
4060 WREG32(INTERRUPT_CNTL, interrupt_cntl);
4061
4062 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
4063 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
4064
4065 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
4066 IH_WPTR_OVERFLOW_CLEAR |
4067 (rb_bufsz << 1));
724c80e1
AD		 4068
4069 if (rdev->wb.enabled)
4070 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
4071
4072 /* set the writeback address whether it's enabled or not */
4073 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
4074 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
d8f60cfc
AD		 4075
4076 WREG32(IH_RB_CNTL, ih_rb_cntl);
4077
4078 /* set rptr, wptr to 0 */
4079 WREG32(IH_RB_RPTR, 0);
4080 WREG32(IH_RB_WPTR, 0);
4081
4082 /* Default settings for IH_CNTL (disabled at first) */
4083 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
4084 /* RPTR_REARM only works if msi's are enabled */
4085 if (rdev->msi_enabled)
4086 ih_cntl |= RPTR_REARM;
d8f60cfc
AD		 4087 WREG32(IH_CNTL, ih_cntl);
4088
4089 /* force the active interrupt state to all disabled */
45f9a39b
AD		 4090 if (rdev->family >= CHIP_CEDAR)
4091 evergreen_disable_interrupt_state(rdev);
4092 else
4093 r600_disable_interrupt_state(rdev);
d8f60cfc 4094
2099810f
DA		 4095 /* at this point everything should be setup correctly to enable master */
4096 pci_set_master(rdev->pdev);
4097
d8f60cfc
AD		 4098 /* enable irqs */
4099 r600_enable_interrupts(rdev);
4100
4101 return ret;
4102}
4103
0c45249f 4104void r600_irq_suspend(struct radeon_device *rdev)
d8f60cfc 4105{
45f9a39b 4106 r600_irq_disable(rdev);
d8f60cfc 4107 r600_rlc_stop(rdev);
0c45249f
JG		 4108}
4109
4110void r600_irq_fini(struct radeon_device *rdev)
4111{
4112 r600_irq_suspend(rdev);
d8f60cfc
AD		 4113 r600_ih_ring_fini(rdev);
4114}
4115
4116int r600_irq_set(struct radeon_device *rdev)
4117{
e0df1ac5
AD		 4118 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4119 u32 mode_int = 0;
4120 u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
2031f77c 4121 u32 grbm_int_cntl = 0;
f122c610 4122 u32 hdmi0, hdmi1;
6f34be50 4123 u32 d1grph = 0, d2grph = 0;
4d75658b 4124 u32 dma_cntl;
4a6369e9 4125 u32 thermal_int = 0;
d8f60cfc 4126
003e69f9 4127 if (!rdev->irq.installed) {
fce7d61b 4128 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
003e69f9
JG		 4129 return -EINVAL;
4130 }
d8f60cfc 4131 /* don't enable anything if the ih is disabled */
79c2bbc5
JG		 4132 if (!rdev->ih.enabled) {
4133 r600_disable_interrupts(rdev);
4134 /* force the active interrupt state to all disabled */
4135 r600_disable_interrupt_state(rdev);
d8f60cfc 4136 return 0;
79c2bbc5 4137 }
d8f60cfc 4138
e0df1ac5
AD		 4139 if (ASIC_IS_DCE3(rdev)) {
4140 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4141 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4142 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4143 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4144 if (ASIC_IS_DCE32(rdev)) {
4145 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4146 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
c6543a6e
RM		 4147 hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4148 hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
f122c610
AD		 4149 } else {
4150 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4151 hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
e0df1ac5
AD		 4152 }
4153 } else {
4154 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4155 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4156 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
f122c610
AD		 4157 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4158 hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
e0df1ac5 4159 }
4a6369e9 4160
4d75658b 4161 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
e0df1ac5 4162
4a6369e9
AD		 4163 if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
4164 thermal_int = RREG32(CG_THERMAL_INT) &
4165 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
66229b20
AD		 4166 } else if (rdev->family >= CHIP_RV770) {
4167 thermal_int = RREG32(RV770_CG_THERMAL_INT) &
4168 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4169 }
4170 if (rdev->irq.dpm_thermal) {
4171 DRM_DEBUG("dpm thermal\n");
4172 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4a6369e9
AD		 4173 }
4174
736fc37f 4175 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
d8f60cfc
AD		 4176 DRM_DEBUG("r600_irq_set: sw int\n");
4177 cp_int_cntl |= RB_INT_ENABLE;
d0f8a854 4178 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
d8f60cfc 4179 }
4d75658b
AD		 4180
4181 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4182 DRM_DEBUG("r600_irq_set: sw int dma\n");
4183 dma_cntl |= TRAP_ENABLE;
4184 }
4185
6f34be50 4186 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4187 atomic_read(&rdev->irq.pflip[0])) {
d8f60cfc
AD		 4188 DRM_DEBUG("r600_irq_set: vblank 0\n");
4189 mode_int |= D1MODE_VBLANK_INT_MASK;
4190 }
6f34be50 4191 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4192 atomic_read(&rdev->irq.pflip[1])) {
d8f60cfc
AD		 4193 DRM_DEBUG("r600_irq_set: vblank 1\n");
4194 mode_int |= D2MODE_VBLANK_INT_MASK;
4195 }
e0df1ac5
AD		 4196 if (rdev->irq.hpd[0]) {
4197 DRM_DEBUG("r600_irq_set: hpd 1\n");
4198 hpd1 |= DC_HPDx_INT_EN;
4199 }
4200 if (rdev->irq.hpd[1]) {
4201 DRM_DEBUG("r600_irq_set: hpd 2\n");
4202 hpd2 |= DC_HPDx_INT_EN;
4203 }
4204 if (rdev->irq.hpd[2]) {
4205 DRM_DEBUG("r600_irq_set: hpd 3\n");
4206 hpd3 |= DC_HPDx_INT_EN;
4207 }
4208 if (rdev->irq.hpd[3]) {
4209 DRM_DEBUG("r600_irq_set: hpd 4\n");
4210 hpd4 |= DC_HPDx_INT_EN;
4211 }
4212 if (rdev->irq.hpd[4]) {
4213 DRM_DEBUG("r600_irq_set: hpd 5\n");
4214 hpd5 |= DC_HPDx_INT_EN;
4215 }
4216 if (rdev->irq.hpd[5]) {
4217 DRM_DEBUG("r600_irq_set: hpd 6\n");
4218 hpd6 |= DC_HPDx_INT_EN;
4219 }
f122c610
AD		 4220 if (rdev->irq.afmt[0]) {
4221 DRM_DEBUG("r600_irq_set: hdmi 0\n");
4222 hdmi0 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
f2594933 4223 }
f122c610
AD		 4224 if (rdev->irq.afmt[1]) {
4225 DRM_DEBUG("r600_irq_set: hdmi 0\n");
4226 hdmi1 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
f2594933 4227 }
d8f60cfc
AD		 4228
4229 WREG32(CP_INT_CNTL, cp_int_cntl);
4d75658b 4230 WREG32(DMA_CNTL, dma_cntl);
d8f60cfc 4231 WREG32(DxMODE_INT_MASK, mode_int);
6f34be50
AD		 4232 WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
4233 WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
2031f77c 4234 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
e0df1ac5
AD		 4235 if (ASIC_IS_DCE3(rdev)) {
4236 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4237 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4238 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4239 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4240 if (ASIC_IS_DCE32(rdev)) {
4241 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4242 WREG32(DC_HPD6_INT_CONTROL, hpd6);
c6543a6e
RM		 4243 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
4244 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
f122c610
AD		 4245 } else {
4246 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
4247 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
e0df1ac5
AD		 4248 }
4249 } else {
4250 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
4251 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
4252 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
f122c610
AD		 4253 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
4254 WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
e0df1ac5 4255 }
4a6369e9
AD		 4256 if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
4257 WREG32(CG_THERMAL_INT, thermal_int);
66229b20
AD		 4258 } else if (rdev->family >= CHIP_RV770) {
4259 WREG32(RV770_CG_THERMAL_INT, thermal_int);
4a6369e9 4260 }
d8f60cfc
AD		 4261
4262 return 0;
4263}
4264
ce580fab 4265static void r600_irq_ack(struct radeon_device *rdev)
d8f60cfc 4266{
e0df1ac5
AD		 4267 u32 tmp;
4268
4269 if (ASIC_IS_DCE3(rdev)) {
6f34be50
AD		 4270 rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
4271 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
4272 rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
f122c610 4273 if (ASIC_IS_DCE32(rdev)) {
c6543a6e
RM		 4274 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
4275 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
f122c610
AD		 4276 } else {
4277 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
4278 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
4279 }
e0df1ac5 4280 } else {
6f34be50
AD		 4281 rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4282 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4283 rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
f122c610
AD		 4284 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
4285 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
6f34be50
AD		 4286 }
4287 rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
4288 rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
4289
4290 if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
4291 WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
4292 if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
4293 WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
4294 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
d8f60cfc 4295 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
6f34be50 4296 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
d8f60cfc 4297 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
6f34be50 4298 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
d8f60cfc 4299 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
6f34be50 4300 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
d8f60cfc 4301 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
6f34be50 4302 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
e0df1ac5
AD		 4303 if (ASIC_IS_DCE3(rdev)) {
4304 tmp = RREG32(DC_HPD1_INT_CONTROL);
4305 tmp |= DC_HPDx_INT_ACK;
4306 WREG32(DC_HPD1_INT_CONTROL, tmp);
4307 } else {
4308 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
4309 tmp |= DC_HPDx_INT_ACK;
4310 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
4311 }
4312 }
6f34be50 4313 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
e0df1ac5
AD		 4314 if (ASIC_IS_DCE3(rdev)) {
4315 tmp = RREG32(DC_HPD2_INT_CONTROL);
4316 tmp |= DC_HPDx_INT_ACK;
4317 WREG32(DC_HPD2_INT_CONTROL, tmp);
4318 } else {
4319 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
4320 tmp |= DC_HPDx_INT_ACK;
4321 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
4322 }
4323 }
6f34be50 4324 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
e0df1ac5
AD		 4325 if (ASIC_IS_DCE3(rdev)) {
4326 tmp = RREG32(DC_HPD3_INT_CONTROL);
4327 tmp |= DC_HPDx_INT_ACK;
4328 WREG32(DC_HPD3_INT_CONTROL, tmp);
4329 } else {
4330 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
4331 tmp |= DC_HPDx_INT_ACK;
4332 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
4333 }
4334 }
6f34be50 4335 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
e0df1ac5
AD		 4336 tmp = RREG32(DC_HPD4_INT_CONTROL);
4337 tmp |= DC_HPDx_INT_ACK;
4338 WREG32(DC_HPD4_INT_CONTROL, tmp);
4339 }
4340 if (ASIC_IS_DCE32(rdev)) {
6f34be50 4341 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
e0df1ac5
AD		 4342 tmp = RREG32(DC_HPD5_INT_CONTROL);
4343 tmp |= DC_HPDx_INT_ACK;
4344 WREG32(DC_HPD5_INT_CONTROL, tmp);
4345 }
6f34be50 4346 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
e0df1ac5
AD		 4347 tmp = RREG32(DC_HPD5_INT_CONTROL);
4348 tmp |= DC_HPDx_INT_ACK;
4349 WREG32(DC_HPD6_INT_CONTROL, tmp);
4350 }
f122c610 4351 if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
c6543a6e 4352 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
f122c610 4353 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
c6543a6e 4354 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
f122c610
AD		 4355 }
4356 if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
c6543a6e 4357 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
f122c610 4358 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
c6543a6e 4359 WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
f2594933
CK		 4360 }
4361 } else {
f122c610
AD		 4362 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4363 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
4364 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4365 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
4366 }
4367 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4368 if (ASIC_IS_DCE3(rdev)) {
4369 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
4370 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4371 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
4372 } else {
4373 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
4374 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4375 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
4376 }
f2594933
CK		 4377 }
4378 }
d8f60cfc
AD		 4379}
4380
4381void r600_irq_disable(struct radeon_device *rdev)
4382{
d8f60cfc
AD		 4383 r600_disable_interrupts(rdev);
4384 /* Wait and acknowledge irq */
4385 mdelay(1);
6f34be50 4386 r600_irq_ack(rdev);
e0df1ac5 4387 r600_disable_interrupt_state(rdev);
d8f60cfc
AD		 4388}
4389
ce580fab 4390static u32 r600_get_ih_wptr(struct radeon_device *rdev)
d8f60cfc
AD		 4391{
4392 u32 wptr, tmp;
3ce0a23d 4393
724c80e1 4394 if (rdev->wb.enabled)
204ae24d 4395 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4396 else
4397 wptr = RREG32(IH_RB_WPTR);
3ce0a23d 4398
d8f60cfc 4399 if (wptr & RB_OVERFLOW) {
7924e5eb
JG		 4400 /* When a ring buffer overflow happen start parsing interrupt
4401 * from the last not overwritten vector (wptr + 16). Hopefully
4402 * this should allow us to catchup.
4403 */
4404 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4405 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4406 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
d8f60cfc
AD		 4407 tmp = RREG32(IH_RB_CNTL);
4408 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4409 WREG32(IH_RB_CNTL, tmp);
4410 }
0c45249f 4411 return (wptr & rdev->ih.ptr_mask);
d8f60cfc 4412}
3ce0a23d 4413
d8f60cfc
AD		 4414/* r600 IV Ring
4415 * Each IV ring entry is 128 bits:
4416 * [7:0] - interrupt source id
4417 * [31:8] - reserved
4418 * [59:32] - interrupt source data
4419 * [127:60] - reserved
4420 *
4421 * The basic interrupt vector entries
4422 * are decoded as follows:
4423 * src_id src_data description
4424 * 1 0 D1 Vblank
4425 * 1 1 D1 Vline
4426 * 5 0 D2 Vblank
4427 * 5 1 D2 Vline
4428 * 19 0 FP Hot plug detection A
4429 * 19 1 FP Hot plug detection B
4430 * 19 2 DAC A auto-detection
4431 * 19 3 DAC B auto-detection
f2594933
CK		 4432 * 21 4 HDMI block A
4433 * 21 5 HDMI block B
d8f60cfc
AD		 4434 * 176 - CP_INT RB
4435 * 177 - CP_INT IB1
4436 * 178 - CP_INT IB2
4437 * 181 - EOP Interrupt
4438 * 233 - GUI Idle
4439 *
4440 * Note, these are based on r600 and may need to be
4441 * adjusted or added to on newer asics
4442 */
4443
4444int r600_irq_process(struct radeon_device *rdev)
4445{
682f1a54
DA		 4446 u32 wptr;
4447 u32 rptr;
d8f60cfc 4448 u32 src_id, src_data;
6f34be50 4449 u32 ring_index;
d4877cf2 4450 bool queue_hotplug = false;
f122c610 4451 bool queue_hdmi = false;
4a6369e9 4452 bool queue_thermal = false;
d8f60cfc 4453
682f1a54 4454 if (!rdev->ih.enabled || rdev->shutdown)
79c2bbc5 4455 return IRQ_NONE;
d8f60cfc 4456
f6a56939
BH		 4457 /* No MSIs, need a dummy read to flush PCI DMAs */
4458 if (!rdev->msi_enabled)
4459 RREG32(IH_RB_WPTR);
4460
682f1a54 4461 wptr = r600_get_ih_wptr(rdev);
d8f60cfc 4462
c20dc369
CK		 4463restart_ih:
4464 /* is somebody else already processing irqs? */
4465 if (atomic_xchg(&rdev->ih.lock, 1))
d8f60cfc 4466 return IRQ_NONE;
d8f60cfc 4467
c20dc369
CK		 4468 rptr = rdev->ih.rptr;
4469 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4470
964f6645
BH		 4471 /* Order reading of wptr vs. reading of IH ring data */
4472 rmb();
4473
d8f60cfc 4474 /* display interrupts */
6f34be50 4475 r600_irq_ack(rdev);
d8f60cfc 4476
d8f60cfc
AD		 4477 while (rptr != wptr) {
4478 /* wptr/rptr are in bytes! */
4479 ring_index = rptr / 4;
4eace7fd
CC		 4480 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4481 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
d8f60cfc
AD		 4482
4483 switch (src_id) {
4484 case 1: /* D1 vblank/vline */
4485 switch (src_data) {
4486 case 0: /* D1 vblank */
6f34be50 4487 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4488 if (rdev->irq.crtc_vblank_int[0]) {
4489 drm_handle_vblank(rdev->ddev, 0);
4490 rdev->pm.vblank_sync = true;
4491 wake_up(&rdev->irq.vblank_queue);
4492 }
736fc37f 4493 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4494 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4495 rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
d8f60cfc
AD		 4496 DRM_DEBUG("IH: D1 vblank\n");
4497 }
4498 break;
4499 case 1: /* D1 vline */
6f34be50
AD		 4500 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
4501 rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
d8f60cfc
AD		 4502 DRM_DEBUG("IH: D1 vline\n");
4503 }
4504 break;
4505 default:
b042589c 4506 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
d8f60cfc
AD		 4507 break;
4508 }
4509 break;
4510 case 5: /* D2 vblank/vline */
4511 switch (src_data) {
4512 case 0: /* D2 vblank */
6f34be50 4513 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4514 if (rdev->irq.crtc_vblank_int[1]) {
4515 drm_handle_vblank(rdev->ddev, 1);
4516 rdev->pm.vblank_sync = true;
4517 wake_up(&rdev->irq.vblank_queue);
4518 }
736fc37f 4519 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4520 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4521 rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
d8f60cfc
AD		 4522 DRM_DEBUG("IH: D2 vblank\n");
4523 }
4524 break;
4525 case 1: /* D1 vline */
6f34be50
AD		 4526 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
4527 rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
d8f60cfc
AD		 4528 DRM_DEBUG("IH: D2 vline\n");
4529 }
4530 break;
4531 default:
b042589c 4532 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
d8f60cfc
AD		 4533 break;
4534 }
4535 break;
e0df1ac5
AD		 4536 case 19: /* HPD/DAC hotplug */
4537 switch (src_data) {
4538 case 0:
6f34be50
AD		 4539 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
4540 rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
d4877cf2
AD		 4541 queue_hotplug = true;
4542 DRM_DEBUG("IH: HPD1\n");
e0df1ac5
AD		 4543 }
4544 break;
4545 case 1:
6f34be50
AD		 4546 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
4547 rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
d4877cf2
AD		 4548 queue_hotplug = true;
4549 DRM_DEBUG("IH: HPD2\n");
e0df1ac5
AD		 4550 }
4551 break;
4552 case 4:
6f34be50
AD		 4553 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
4554 rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
d4877cf2
AD		 4555 queue_hotplug = true;
4556 DRM_DEBUG("IH: HPD3\n");
e0df1ac5
AD		 4557 }
4558 break;
4559 case 5:
6f34be50
AD		 4560 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
4561 rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
d4877cf2
AD		 4562 queue_hotplug = true;
4563 DRM_DEBUG("IH: HPD4\n");
e0df1ac5
AD		 4564 }
4565 break;
4566 case 10:
6f34be50
AD		 4567 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
4568 rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
d4877cf2
AD		 4569 queue_hotplug = true;
4570 DRM_DEBUG("IH: HPD5\n");
e0df1ac5
AD		 4571 }
4572 break;
4573 case 12:
6f34be50
AD		 4574 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
4575 rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
d4877cf2
AD		 4576 queue_hotplug = true;
4577 DRM_DEBUG("IH: HPD6\n");
e0df1ac5
AD		 4578 }
4579 break;
4580 default:
b042589c 4581 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
e0df1ac5
AD		 4582 break;
4583 }
4584 break;
f122c610
AD		 4585 case 21: /* hdmi */
4586 switch (src_data) {
4587 case 4:
4588 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4589 rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4590 queue_hdmi = true;
4591 DRM_DEBUG("IH: HDMI0\n");
4592 }
4593 break;
4594 case 5:
4595 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4596 rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4597 queue_hdmi = true;
4598 DRM_DEBUG("IH: HDMI1\n");
4599 }
4600 break;
4601 default:
4602 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4603 break;
4604 }
f2594933 4605 break;
d8f60cfc
AD		 4606 case 176: /* CP_INT in ring buffer */
4607 case 177: /* CP_INT in IB1 */
4608 case 178: /* CP_INT in IB2 */
4609 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 4610 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
d8f60cfc
AD		 4611 break;
4612 case 181: /* CP EOP event */
4613 DRM_DEBUG("IH: CP EOP\n");
7465280c 4614 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
d8f60cfc 4615 break;
4d75658b
AD		 4616 case 224: /* DMA trap event */
4617 DRM_DEBUG("IH: DMA trap\n");
4618 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4619 break;
4a6369e9
AD		 4620 case 230: /* thermal low to high */
4621 DRM_DEBUG("IH: thermal low to high\n");
4622 rdev->pm.dpm.thermal.high_to_low = false;
4623 queue_thermal = true;
4624 break;
4625 case 231: /* thermal high to low */
4626 DRM_DEBUG("IH: thermal high to low\n");
4627 rdev->pm.dpm.thermal.high_to_low = true;
4628 queue_thermal = true;
4629 break;
2031f77c 4630 case 233: /* GUI IDLE */
303c805c 4631 DRM_DEBUG("IH: GUI idle\n");
2031f77c 4632 break;
d8f60cfc 4633 default:
b042589c 4634 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
d8f60cfc
AD		 4635 break;
4636 }
4637
4638 /* wptr/rptr are in bytes! */
0c45249f
JG		 4639 rptr += 16;
4640 rptr &= rdev->ih.ptr_mask;
d8f60cfc 4641 }
d4877cf2 4642 if (queue_hotplug)
32c87fca 4643 schedule_work(&rdev->hotplug_work);
f122c610
AD		 4644 if (queue_hdmi)
4645 schedule_work(&rdev->audio_work);
4a6369e9
AD		 4646 if (queue_thermal && rdev->pm.dpm_enabled)
4647 schedule_work(&rdev->pm.dpm.thermal.work);
d8f60cfc
AD		 4648 rdev->ih.rptr = rptr;
4649 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 4650 atomic_set(&rdev->ih.lock, 0);
4651
4652 /* make sure wptr hasn't changed while processing */
4653 wptr = r600_get_ih_wptr(rdev);
4654 if (wptr != rptr)
4655 goto restart_ih;
4656
d8f60cfc
AD		 4657 return IRQ_HANDLED;
4658}
3ce0a23d
JG		 4659
4660/*
4661 * Debugfs info
4662 */
4663#if defined(CONFIG_DEBUG_FS)
4664
3ce0a23d
JG		 4665static int r600_debugfs_mc_info(struct seq_file *m, void *data)
4666{
4667 struct drm_info_node *node = (struct drm_info_node *) m->private;
4668 struct drm_device *dev = node->minor->dev;
4669 struct radeon_device *rdev = dev->dev_private;
4670
4671 DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
4672 DREG32_SYS(m, rdev, VM_L2_STATUS);
4673 return 0;
4674}
4675
4676static struct drm_info_list r600_mc_info_list[] = {
4677 {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
3ce0a23d
JG		 4678};
4679#endif
4680
4681int r600_debugfs_mc_info_init(struct radeon_device *rdev)
4682{
4683#if defined(CONFIG_DEBUG_FS)
4684 return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
4685#else
4686 return 0;
4687#endif
771fe6b9 4688}
062b389c
JG		 4689
4690/**
4691 * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
4692 * rdev: radeon device structure
4693 * bo: buffer object struct which userspace is waiting for idle
4694 *
4695 * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
4696 * through ring buffer, this leads to corruption in rendering, see
4697 * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
4698 * directly perform HDP flush by writing register through MMIO.
4699 */
4700void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
4701{
812d0469 4702 /* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
f3886f85
AD		 4703 * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
4704 * This seems to cause problems on some AGP cards. Just use the old
4705 * method for them.
812d0469 4706 */
e488459a 4707 if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
f3886f85 4708 rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
87cbf8f2 4709 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
812d0469
AD		 4710 u32 tmp;
4711
4712 WREG32(HDP_DEBUG1, 0);
4713 tmp = readl((void __iomem *)ptr);
4714 } else
4715 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
062b389c 4716}
3313e3d4
AD		 4717
4718void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
4719{
d5445a17 4720 u32 link_width_cntl, mask;
3313e3d4
AD		 4721
4722 if (rdev->flags & RADEON_IS_IGP)
4723 return;
4724
4725 if (!(rdev->flags & RADEON_IS_PCIE))
4726 return;
4727
4728 /* x2 cards have a special sequence */
4729 if (ASIC_IS_X2(rdev))
4730 return;
4731
d5445a17 4732 radeon_gui_idle(rdev);
3313e3d4
AD		 4733
4734 switch (lanes) {
4735 case 0:
4736 mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
4737 break;
4738 case 1:
4739 mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
4740 break;
4741 case 2:
4742 mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
4743 break;
4744 case 4:
4745 mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
4746 break;
4747 case 8:
4748 mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
4749 break;
4750 case 12:
d5445a17 4751 /* not actually supported */
3313e3d4
AD		 4752 mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
4753 break;
4754 case 16:
3313e3d4
AD		 4755 mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
4756 break;
d5445a17
AD		 4757 default:
4758 DRM_ERROR("invalid pcie lane request: %d\n", lanes);
4759 return;
3313e3d4
AD		 4760 }
4761
492d2b61 4762 link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
d5445a17
AD		 4763 link_width_cntl &= ~RADEON_PCIE_LC_LINK_WIDTH_MASK;
4764 link_width_cntl |= mask << RADEON_PCIE_LC_LINK_WIDTH_SHIFT;
4765 link_width_cntl |= (RADEON_PCIE_LC_RECONFIG_NOW |
4766 R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
3313e3d4 4767
492d2b61 4768 WREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3313e3d4
AD		 4769}
4770
4771int r600_get_pcie_lanes(struct radeon_device *rdev)
4772{
4773 u32 link_width_cntl;
4774
4775 if (rdev->flags & RADEON_IS_IGP)
4776 return 0;
4777
4778 if (!(rdev->flags & RADEON_IS_PCIE))
4779 return 0;
4780
4781 /* x2 cards have a special sequence */
4782 if (ASIC_IS_X2(rdev))
4783 return 0;
4784
d5445a17 4785 radeon_gui_idle(rdev);
3313e3d4 4786
492d2b61 4787 link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
3313e3d4
AD		 4788
4789 switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
3313e3d4
AD		 4790 case RADEON_PCIE_LC_LINK_WIDTH_X1:
4791 return 1;
4792 case RADEON_PCIE_LC_LINK_WIDTH_X2:
4793 return 2;
4794 case RADEON_PCIE_LC_LINK_WIDTH_X4:
4795 return 4;
4796 case RADEON_PCIE_LC_LINK_WIDTH_X8:
4797 return 8;
d5445a17
AD		 4798 case RADEON_PCIE_LC_LINK_WIDTH_X12:
4799 /* not actually supported */
4800 return 12;
4801 case RADEON_PCIE_LC_LINK_WIDTH_X0:
3313e3d4
AD		 4802 case RADEON_PCIE_LC_LINK_WIDTH_X16:
4803 default:
4804 return 16;
4805 }
4806}
4807
9e46a48d
AD		 4808static void r600_pcie_gen2_enable(struct radeon_device *rdev)
4809{
4810 u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
4811 u16 link_cntl2;
4812
d42dd579
AD		 4813 if (radeon_pcie_gen2 == 0)
4814 return;
4815
9e46a48d
AD		 4816 if (rdev->flags & RADEON_IS_IGP)
4817 return;
4818
4819 if (!(rdev->flags & RADEON_IS_PCIE))
4820 return;
4821
4822 /* x2 cards have a special sequence */
4823 if (ASIC_IS_X2(rdev))
4824 return;
4825
4826 /* only RV6xx+ chips are supported */
4827 if (rdev->family <= CHIP_R600)
4828 return;
4829
7e0e4196
KSS		 4830 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
4831 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 4832 return;
4833
492d2b61 4834 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 4835 if (speed_cntl & LC_CURRENT_DATA_RATE) {
4836 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
4837 return;
4838 }
4839
197bbb3d
DA		 4840 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
4841
9e46a48d
AD		 4842 /* 55 nm r6xx asics */
4843 if ((rdev->family == CHIP_RV670) ||
4844 (rdev->family == CHIP_RV620) ||
4845 (rdev->family == CHIP_RV635)) {
4846 /* advertise upconfig capability */
492d2b61 4847 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 4848 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61
AD		 4849 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4850 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 4851 if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
4852 lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
4853 link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
4854 LC_RECONFIG_ARC_MISSING_ESCAPE);
4855 link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
492d2b61 4856 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 4857 } else {
4858 link_width_cntl |= LC_UPCONFIGURE_DIS;
492d2b61 4859 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 4860 }
4861 }
4862
492d2b61 4863 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d
AD		 4864 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
4865 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
4866
4867 /* 55 nm r6xx asics */
4868 if ((rdev->family == CHIP_RV670) ||
4869 (rdev->family == CHIP_RV620) ||
4870 (rdev->family == CHIP_RV635)) {
4871 WREG32(MM_CFGREGS_CNTL, 0x8);
4872 link_cntl2 = RREG32(0x4088);
4873 WREG32(MM_CFGREGS_CNTL, 0);
4874 /* not supported yet */
4875 if (link_cntl2 & SELECTABLE_DEEMPHASIS)
4876 return;
4877 }
4878
4879 speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
4880 speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
4881 speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
4882 speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
4883 speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
492d2b61 4884 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 4885
4886 tmp = RREG32(0x541c);
4887 WREG32(0x541c, tmp | 0x8);
4888 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
4889 link_cntl2 = RREG16(0x4088);
4890 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
4891 link_cntl2 |= 0x2;
4892 WREG16(0x4088, link_cntl2);
4893 WREG32(MM_CFGREGS_CNTL, 0);
4894
4895 if ((rdev->family == CHIP_RV670) ||
4896 (rdev->family == CHIP_RV620) ||
4897 (rdev->family == CHIP_RV635)) {
492d2b61 4898 training_cntl = RREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL);
9e46a48d 4899 training_cntl &= ~LC_POINT_7_PLUS_EN;
492d2b61 4900 WREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL, training_cntl);
9e46a48d 4901 } else {
492d2b61 4902 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 4903 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 4904 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 4905 }
4906
492d2b61 4907 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 4908 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 4909 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 4910
4911 } else {
492d2b61 4912 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 4913 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
4914 if (1)
4915 link_width_cntl |= LC_UPCONFIGURE_DIS;
4916 else
4917 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 4918 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 4919 }
4920}
6759a0a7
MO		 4921
4922/**
d0418894 4923 * r600_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 4924 *
4925 * @rdev: radeon_device pointer
4926 *
4927 * Fetches a GPU clock counter snapshot (R6xx-cayman).
4928 * Returns the 64 bit clock counter snapshot.
4929 */
d0418894 4930uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 4931{
4932 uint64_t clock;
4933
4934 mutex_lock(&rdev->gpu_clock_mutex);
4935 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4936 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
4937 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4938 mutex_unlock(&rdev->gpu_clock_mutex);
4939 return clock;
4940}
ubuntu-zesty-kernel mirror