]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/gpu/drm/i915/intel_pm.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Introduce intel_fb_obj() macro
[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / i915 / intel_pm.c
CommitLineData
85208be0
ED		 1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
2b4e57bd 28#include <linux/cpufreq.h>
85208be0
ED		 29#include "i915_drv.h"
30#include "intel_drv.h"
eb48eb00
DV		 31#include "../../../platform/x86/intel_ips.h"
32#include <linux/module.h>
f9dcb0df 33#include <linux/vgaarb.h>
f4db9321 34#include <drm/i915_powerwell.h>
8a187455 35#include <linux/pm_runtime.h>
85208be0 36
dc39fff7
BW		 37/**
38 * RC6 is a special power stage which allows the GPU to enter an very
39 * low-voltage mode when idle, using down to 0V while at this stage. This
40 * stage is entered automatically when the GPU is idle when RC6 support is
41 * enabled, and as soon as new workload arises GPU wakes up automatically as well.
42 *
43 * There are different RC6 modes available in Intel GPU, which differentiate
44 * among each other with the latency required to enter and leave RC6 and
45 * voltage consumed by the GPU in different states.
46 *
47 * The combination of the following flags define which states GPU is allowed
48 * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
49 * RC6pp is deepest RC6. Their support by hardware varies according to the
50 * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
51 * which brings the most power savings; deeper states save more power, but
52 * require higher latency to switch to and wake up.
53 */
54#define INTEL_RC6_ENABLE (1<<0)
55#define INTEL_RC6p_ENABLE (1<<1)
56#define INTEL_RC6pp_ENABLE (1<<2)
57
f6750b3c
ED		 58/* FBC, or Frame Buffer Compression, is a technique employed to compress the
59 * framebuffer contents in-memory, aiming at reducing the required bandwidth
60 * during in-memory transfers and, therefore, reduce the power packet.
85208be0 61 *
f6750b3c
ED		 62 * The benefits of FBC are mostly visible with solid backgrounds and
63 * variation-less patterns.
85208be0 64 *
f6750b3c
ED		 65 * FBC-related functionality can be enabled by the means of the
66 * i915.i915_enable_fbc parameter
85208be0
ED		 67 */
68
1fa61106 69static void i8xx_disable_fbc(struct drm_device *dev)
85208be0
ED		 70{
71 struct drm_i915_private *dev_priv = dev->dev_private;
72 u32 fbc_ctl;
73
74 /* Disable compression */
75 fbc_ctl = I915_READ(FBC_CONTROL);
76 if ((fbc_ctl & FBC_CTL_EN) == 0)
77 return;
78
79 fbc_ctl &= ~FBC_CTL_EN;
80 I915_WRITE(FBC_CONTROL, fbc_ctl);
81
82 /* Wait for compressing bit to clear */
83 if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
84 DRM_DEBUG_KMS("FBC idle timed out\n");
85 return;
86 }
87
88 DRM_DEBUG_KMS("disabled FBC\n");
89}
90
993495ae 91static void i8xx_enable_fbc(struct drm_crtc *crtc)
85208be0
ED		 92{
93 struct drm_device *dev = crtc->dev;
94 struct drm_i915_private *dev_priv = dev->dev_private;
f4510a27 95 struct drm_framebuffer *fb = crtc->primary->fb;
85208be0
ED		 96 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
97 struct drm_i915_gem_object *obj = intel_fb->obj;
98 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
99 int cfb_pitch;
7f2cf220 100 int i;
159f9875 101 u32 fbc_ctl;
85208be0 102
5c3fe8b0 103 cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE;
85208be0
ED		 104 if (fb->pitches[0] < cfb_pitch)
105 cfb_pitch = fb->pitches[0];
106
42a430f5
VS		 107 /* FBC_CTL wants 32B or 64B units */
108 if (IS_GEN2(dev))
109 cfb_pitch = (cfb_pitch / 32) - 1;
110 else
111 cfb_pitch = (cfb_pitch / 64) - 1;
85208be0
ED		 112
113 /* Clear old tags */
114 for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
115 I915_WRITE(FBC_TAG + (i * 4), 0);
116
159f9875
VS		 117 if (IS_GEN4(dev)) {
118 u32 fbc_ctl2;
119
120 /* Set it up... */
121 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
7f2cf220 122 fbc_ctl2 |= FBC_CTL_PLANE(intel_crtc->plane);
159f9875
VS		 123 I915_WRITE(FBC_CONTROL2, fbc_ctl2);
124 I915_WRITE(FBC_FENCE_OFF, crtc->y);
125 }
85208be0
ED		 126
127 /* enable it... */
993495ae
VS		 128 fbc_ctl = I915_READ(FBC_CONTROL);
129 fbc_ctl &= 0x3fff << FBC_CTL_INTERVAL_SHIFT;
130 fbc_ctl |= FBC_CTL_EN | FBC_CTL_PERIODIC;
85208be0
ED		 131 if (IS_I945GM(dev))
132 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
133 fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
85208be0
ED		 134 fbc_ctl |= obj->fence_reg;
135 I915_WRITE(FBC_CONTROL, fbc_ctl);
136
5cd5410e 137 DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c\n",
84f44ce7 138 cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
85208be0
ED		 139}
140
1fa61106 141static bool i8xx_fbc_enabled(struct drm_device *dev)
85208be0
ED		 142{
143 struct drm_i915_private *dev_priv = dev->dev_private;
144
145 return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
146}
147
993495ae 148static void g4x_enable_fbc(struct drm_crtc *crtc)
85208be0
ED		 149{
150 struct drm_device *dev = crtc->dev;
151 struct drm_i915_private *dev_priv = dev->dev_private;
f4510a27 152 struct drm_framebuffer *fb = crtc->primary->fb;
85208be0
ED		 153 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
154 struct drm_i915_gem_object *obj = intel_fb->obj;
155 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
85208be0
ED		 156 u32 dpfc_ctl;
157
3fa2e0ee
VS		 158 dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane) | DPFC_SR_EN;
159 if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
160 dpfc_ctl |= DPFC_CTL_LIMIT_2X;
161 else
162 dpfc_ctl |= DPFC_CTL_LIMIT_1X;
85208be0 163 dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
85208be0 164
85208be0
ED		 165 I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
166
167 /* enable it... */
fe74c1a5 168 I915_WRITE(DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
85208be0 169
84f44ce7 170 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
85208be0
ED		 171}
172
1fa61106 173static void g4x_disable_fbc(struct drm_device *dev)
85208be0
ED		 174{
175 struct drm_i915_private *dev_priv = dev->dev_private;
176 u32 dpfc_ctl;
177
178 /* Disable compression */
179 dpfc_ctl = I915_READ(DPFC_CONTROL);
180 if (dpfc_ctl & DPFC_CTL_EN) {
181 dpfc_ctl &= ~DPFC_CTL_EN;
182 I915_WRITE(DPFC_CONTROL, dpfc_ctl);
183
184 DRM_DEBUG_KMS("disabled FBC\n");
185 }
186}
187
1fa61106 188static bool g4x_fbc_enabled(struct drm_device *dev)
85208be0
ED		 189{
190 struct drm_i915_private *dev_priv = dev->dev_private;
191
192 return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
193}
194
195static void sandybridge_blit_fbc_update(struct drm_device *dev)
196{
197 struct drm_i915_private *dev_priv = dev->dev_private;
198 u32 blt_ecoskpd;
199
200 /* Make sure blitter notifies FBC of writes */
940aece4
D		 201
202 /* Blitter is part of Media powerwell on VLV. No impact of
203 * his param in other platforms for now */
204 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_MEDIA);
c8d9a590 205
85208be0
ED		 206 blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
207 blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY <<
208 GEN6_BLITTER_LOCK_SHIFT;
209 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
210 blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY;
211 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
212 blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
213 GEN6_BLITTER_LOCK_SHIFT);
214 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
215 POSTING_READ(GEN6_BLITTER_ECOSKPD);
c8d9a590 216
940aece4 217 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_MEDIA);
85208be0
ED		 218}
219
993495ae 220static void ironlake_enable_fbc(struct drm_crtc *crtc)
85208be0
ED		 221{
222 struct drm_device *dev = crtc->dev;
223 struct drm_i915_private *dev_priv = dev->dev_private;
f4510a27 224 struct drm_framebuffer *fb = crtc->primary->fb;
85208be0
ED		 225 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
226 struct drm_i915_gem_object *obj = intel_fb->obj;
227 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
85208be0
ED		 228 u32 dpfc_ctl;
229
46f3dab9 230 dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane);
3fa2e0ee 231 if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
5e59f717
BW		 232 dev_priv->fbc.threshold++;
233
234 switch (dev_priv->fbc.threshold) {
235 case 4:
236 case 3:
237 dpfc_ctl |= DPFC_CTL_LIMIT_4X;
238 break;
239 case 2:
3fa2e0ee 240 dpfc_ctl |= DPFC_CTL_LIMIT_2X;
5e59f717
BW		 241 break;
242 case 1:
3fa2e0ee 243 dpfc_ctl |= DPFC_CTL_LIMIT_1X;
5e59f717
BW		 244 break;
245 }
d629336b
VS		 246 dpfc_ctl |= DPFC_CTL_FENCE_EN;
247 if (IS_GEN5(dev))
248 dpfc_ctl |= obj->fence_reg;
85208be0 249
85208be0 250 I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
f343c5f6 251 I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) | ILK_FBC_RT_VALID);
85208be0
ED		 252 /* enable it... */
253 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
254
255 if (IS_GEN6(dev)) {
256 I915_WRITE(SNB_DPFC_CTL_SA,
257 SNB_CPU_FENCE_ENABLE | obj->fence_reg);
258 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
259 sandybridge_blit_fbc_update(dev);
260 }
261
84f44ce7 262 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
85208be0
ED		 263}
264
1fa61106 265static void ironlake_disable_fbc(struct drm_device *dev)
85208be0
ED		 266{
267 struct drm_i915_private *dev_priv = dev->dev_private;
268 u32 dpfc_ctl;
269
270 /* Disable compression */
271 dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
272 if (dpfc_ctl & DPFC_CTL_EN) {
273 dpfc_ctl &= ~DPFC_CTL_EN;
274 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
275
276 DRM_DEBUG_KMS("disabled FBC\n");
277 }
278}
279
1fa61106 280static bool ironlake_fbc_enabled(struct drm_device *dev)
85208be0
ED		 281{
282 struct drm_i915_private *dev_priv = dev->dev_private;
283
284 return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
285}
286
993495ae 287static void gen7_enable_fbc(struct drm_crtc *crtc)
abe959c7
RV		 288{
289 struct drm_device *dev = crtc->dev;
290 struct drm_i915_private *dev_priv = dev->dev_private;
f4510a27 291 struct drm_framebuffer *fb = crtc->primary->fb;
abe959c7
RV		 292 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
293 struct drm_i915_gem_object *obj = intel_fb->obj;
294 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3fa2e0ee 295 u32 dpfc_ctl;
abe959c7 296
3fa2e0ee
VS		 297 dpfc_ctl = IVB_DPFC_CTL_PLANE(intel_crtc->plane);
298 if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
5e59f717
BW		 299 dev_priv->fbc.threshold++;
300
301 switch (dev_priv->fbc.threshold) {
302 case 4:
303 case 3:
304 dpfc_ctl |= DPFC_CTL_LIMIT_4X;
305 break;
306 case 2:
3fa2e0ee 307 dpfc_ctl |= DPFC_CTL_LIMIT_2X;
5e59f717
BW		 308 break;
309 case 1:
3fa2e0ee 310 dpfc_ctl |= DPFC_CTL_LIMIT_1X;
5e59f717
BW		 311 break;
312 }
313
3fa2e0ee
VS		 314 dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN;
315
316 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
abe959c7 317
891348b2 318 if (IS_IVYBRIDGE(dev)) {
7dd23ba0 319 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
2adb6db8
VS		 320 I915_WRITE(ILK_DISPLAY_CHICKEN1,
321 I915_READ(ILK_DISPLAY_CHICKEN1) |
322 ILK_FBCQ_DIS);
28554164 323 } else {
2adb6db8 324 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
8f670bb1
VS		 325 I915_WRITE(CHICKEN_PIPESL_1(intel_crtc->pipe),
326 I915_READ(CHICKEN_PIPESL_1(intel_crtc->pipe)) |
327 HSW_FBCQ_DIS);
891348b2 328 }
b74ea102 329
abe959c7
RV		 330 I915_WRITE(SNB_DPFC_CTL_SA,
331 SNB_CPU_FENCE_ENABLE | obj->fence_reg);
332 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
333
334 sandybridge_blit_fbc_update(dev);
335
b19870ee 336 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
abe959c7
RV		 337}
338
85208be0
ED		 339bool intel_fbc_enabled(struct drm_device *dev)
340{
341 struct drm_i915_private *dev_priv = dev->dev_private;
342
343 if (!dev_priv->display.fbc_enabled)
344 return false;
345
346 return dev_priv->display.fbc_enabled(dev);
347}
348
349static void intel_fbc_work_fn(struct work_struct *__work)
350{
351 struct intel_fbc_work *work =
352 container_of(to_delayed_work(__work),
353 struct intel_fbc_work, work);
354 struct drm_device *dev = work->crtc->dev;
355 struct drm_i915_private *dev_priv = dev->dev_private;
356
357 mutex_lock(&dev->struct_mutex);
5c3fe8b0 358 if (work == dev_priv->fbc.fbc_work) {
85208be0
ED		 359 /* Double check that we haven't switched fb without cancelling
360 * the prior work.
361 */
f4510a27 362 if (work->crtc->primary->fb == work->fb) {
993495ae 363 dev_priv->display.enable_fbc(work->crtc);
85208be0 364
5c3fe8b0 365 dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane;
f4510a27 366 dev_priv->fbc.fb_id = work->crtc->primary->fb->base.id;
5c3fe8b0 367 dev_priv->fbc.y = work->crtc->y;
85208be0
ED		 368 }
369
5c3fe8b0 370 dev_priv->fbc.fbc_work = NULL;
85208be0
ED		 371 }
372 mutex_unlock(&dev->struct_mutex);
373
374 kfree(work);
375}
376
377static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
378{
5c3fe8b0 379 if (dev_priv->fbc.fbc_work == NULL)
85208be0
ED		 380 return;
381
382 DRM_DEBUG_KMS("cancelling pending FBC enable\n");
383
384 /* Synchronisation is provided by struct_mutex and checking of
5c3fe8b0 385 * dev_priv->fbc.fbc_work, so we can perform the cancellation
85208be0
ED		 386 * entirely asynchronously.
387 */
5c3fe8b0 388 if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work))
85208be0 389 /* tasklet was killed before being run, clean up */
5c3fe8b0 390 kfree(dev_priv->fbc.fbc_work);
85208be0
ED		 391
392 /* Mark the work as no longer wanted so that if it does
393 * wake-up (because the work was already running and waiting
394 * for our mutex), it will discover that is no longer
395 * necessary to run.
396 */
5c3fe8b0 397 dev_priv->fbc.fbc_work = NULL;
85208be0
ED		 398}
399
993495ae 400static void intel_enable_fbc(struct drm_crtc *crtc)
85208be0
ED		 401{
402 struct intel_fbc_work *work;
403 struct drm_device *dev = crtc->dev;
404 struct drm_i915_private *dev_priv = dev->dev_private;
405
406 if (!dev_priv->display.enable_fbc)
407 return;
408
409 intel_cancel_fbc_work(dev_priv);
410
b14c5679 411 work = kzalloc(sizeof(*work), GFP_KERNEL);
85208be0 412 if (work == NULL) {
6cdcb5e7 413 DRM_ERROR("Failed to allocate FBC work structure\n");
993495ae 414 dev_priv->display.enable_fbc(crtc);
85208be0
ED		 415 return;
416 }
417
418 work->crtc = crtc;
f4510a27 419 work->fb = crtc->primary->fb;
85208be0
ED		 420 INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
421
5c3fe8b0 422 dev_priv->fbc.fbc_work = work;
85208be0 423
85208be0
ED		 424 /* Delay the actual enabling to let pageflipping cease and the
425 * display to settle before starting the compression. Note that
426 * this delay also serves a second purpose: it allows for a
427 * vblank to pass after disabling the FBC before we attempt
428 * to modify the control registers.
429 *
430 * A more complicated solution would involve tracking vblanks
431 * following the termination of the page-flipping sequence
432 * and indeed performing the enable as a co-routine and not
433 * waiting synchronously upon the vblank.
7457d617
DL		 434 *
435 * WaFbcWaitForVBlankBeforeEnable:ilk,snb
85208be0
ED		 436 */
437 schedule_delayed_work(&work->work, msecs_to_jiffies(50));
438}
439
440void intel_disable_fbc(struct drm_device *dev)
441{
442 struct drm_i915_private *dev_priv = dev->dev_private;
443
444 intel_cancel_fbc_work(dev_priv);
445
446 if (!dev_priv->display.disable_fbc)
447 return;
448
449 dev_priv->display.disable_fbc(dev);
5c3fe8b0 450 dev_priv->fbc.plane = -1;
85208be0
ED		 451}
452
29ebf90f
CW		 453static bool set_no_fbc_reason(struct drm_i915_private *dev_priv,
454 enum no_fbc_reason reason)
455{
456 if (dev_priv->fbc.no_fbc_reason == reason)
457 return false;
458
459 dev_priv->fbc.no_fbc_reason = reason;
460 return true;
461}
462
85208be0
ED		 463/**
464 * intel_update_fbc - enable/disable FBC as needed
465 * @dev: the drm_device
466 *
467 * Set up the framebuffer compression hardware at mode set time. We
468 * enable it if possible:
469 * - plane A only (on pre-965)
470 * - no pixel mulitply/line duplication
471 * - no alpha buffer discard
472 * - no dual wide
f85da868 473 * - framebuffer <= max_hdisplay in width, max_vdisplay in height
85208be0
ED		 474 *
475 * We can't assume that any compression will take place (worst case),
476 * so the compressed buffer has to be the same size as the uncompressed
477 * one. It also must reside (along with the line length buffer) in
478 * stolen memory.
479 *
480 * We need to enable/disable FBC on a global basis.
481 */
482void intel_update_fbc(struct drm_device *dev)
483{
484 struct drm_i915_private *dev_priv = dev->dev_private;
485 struct drm_crtc *crtc = NULL, *tmp_crtc;
486 struct intel_crtc *intel_crtc;
487 struct drm_framebuffer *fb;
488 struct intel_framebuffer *intel_fb;
489 struct drm_i915_gem_object *obj;
ef644fda 490 const struct drm_display_mode *adjusted_mode;
37327abd 491 unsigned int max_width, max_height;
85208be0 492
3a77c4c4 493 if (!HAS_FBC(dev)) {
29ebf90f 494 set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED);
85208be0 495 return;
29ebf90f 496 }
85208be0 497
d330a953 498 if (!i915.powersave) {
29ebf90f
CW		 499 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
500 DRM_DEBUG_KMS("fbc disabled per module param\n");
85208be0 501 return;
29ebf90f 502 }
85208be0
ED		 503
504 /*
505 * If FBC is already on, we just have to verify that we can
506 * keep it that way...
507 * Need to disable if:
508 * - more than one pipe is active
509 * - changing FBC params (stride, fence, mode)
510 * - new fb is too large to fit in compressed buffer
511 * - going to an unsupported config (interlace, pixel multiply, etc.)
512 */
70e1e0ec 513 for_each_crtc(dev, tmp_crtc) {
3490ea5d 514 if (intel_crtc_active(tmp_crtc) &&
4c445e0e 515 to_intel_crtc(tmp_crtc)->primary_enabled) {
85208be0 516 if (crtc) {
29ebf90f
CW		 517 if (set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES))
518 DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
85208be0
ED		 519 goto out_disable;
520 }
521 crtc = tmp_crtc;
522 }
523 }
524
f4510a27 525 if (!crtc || crtc->primary->fb == NULL) {
29ebf90f
CW		 526 if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT))
527 DRM_DEBUG_KMS("no output, disabling\n");
85208be0
ED		 528 goto out_disable;
529 }
530
531 intel_crtc = to_intel_crtc(crtc);
f4510a27 532 fb = crtc->primary->fb;
85208be0
ED		 533 intel_fb = to_intel_framebuffer(fb);
534 obj = intel_fb->obj;
ef644fda 535 adjusted_mode = &intel_crtc->config.adjusted_mode;
85208be0 536
0368920e 537 if (i915.enable_fbc < 0) {
29ebf90f
CW		 538 if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT))
539 DRM_DEBUG_KMS("disabled per chip default\n");
8a5729a3 540 goto out_disable;
85208be0 541 }
d330a953 542 if (!i915.enable_fbc) {
29ebf90f
CW		 543 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
544 DRM_DEBUG_KMS("fbc disabled per module param\n");
85208be0
ED		 545 goto out_disable;
546 }
ef644fda
VS		 547 if ((adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) ||
548 (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)) {
29ebf90f
CW		 549 if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
550 DRM_DEBUG_KMS("mode incompatible with compression, "
551 "disabling\n");
85208be0
ED		 552 goto out_disable;
553 }
f85da868 554
032843a5
DS		 555 if (INTEL_INFO(dev)->gen >= 8 || IS_HASWELL(dev)) {
556 max_width = 4096;
557 max_height = 4096;
558 } else if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
37327abd
VS		 559 max_width = 4096;
560 max_height = 2048;
f85da868 561 } else {
37327abd
VS		 562 max_width = 2048;
563 max_height = 1536;
f85da868 564 }
37327abd
VS		 565 if (intel_crtc->config.pipe_src_w > max_width ||
566 intel_crtc->config.pipe_src_h > max_height) {
29ebf90f
CW		 567 if (set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE))
568 DRM_DEBUG_KMS("mode too large for compression, disabling\n");
85208be0
ED		 569 goto out_disable;
570 }
8f94d24b 571 if ((INTEL_INFO(dev)->gen < 4 || HAS_DDI(dev)) &&
c5a44aa0 572 intel_crtc->plane != PLANE_A) {
29ebf90f 573 if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE))
c5a44aa0 574 DRM_DEBUG_KMS("plane not A, disabling compression\n");
85208be0
ED		 575 goto out_disable;
576 }
577
578 /* The use of a CPU fence is mandatory in order to detect writes
579 * by the CPU to the scanout and trigger updates to the FBC.
580 */
581 if (obj->tiling_mode != I915_TILING_X ||
582 obj->fence_reg == I915_FENCE_REG_NONE) {
29ebf90f
CW		 583 if (set_no_fbc_reason(dev_priv, FBC_NOT_TILED))
584 DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
85208be0
ED		 585 goto out_disable;
586 }
587
588 /* If the kernel debugger is active, always disable compression */
589 if (in_dbg_master())
590 goto out_disable;
591
5e59f717
BW		 592 if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size,
593 drm_format_plane_cpp(fb->pixel_format, 0))) {
29ebf90f
CW		 594 if (set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL))
595 DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
11be49eb
CW		 596 goto out_disable;
597 }
598
85208be0
ED		 599 /* If the scanout has not changed, don't modify the FBC settings.
600 * Note that we make the fundamental assumption that the fb->obj
601 * cannot be unpinned (and have its GTT offset and fence revoked)
602 * without first being decoupled from the scanout and FBC disabled.
603 */
5c3fe8b0
BW		 604 if (dev_priv->fbc.plane == intel_crtc->plane &&
605 dev_priv->fbc.fb_id == fb->base.id &&
606 dev_priv->fbc.y == crtc->y)
85208be0
ED		 607 return;
608
609 if (intel_fbc_enabled(dev)) {
610 /* We update FBC along two paths, after changing fb/crtc
611 * configuration (modeswitching) and after page-flipping
612 * finishes. For the latter, we know that not only did
613 * we disable the FBC at the start of the page-flip
614 * sequence, but also more than one vblank has passed.
615 *
616 * For the former case of modeswitching, it is possible
617 * to switch between two FBC valid configurations
618 * instantaneously so we do need to disable the FBC
619 * before we can modify its control registers. We also
620 * have to wait for the next vblank for that to take
621 * effect. However, since we delay enabling FBC we can
622 * assume that a vblank has passed since disabling and
623 * that we can safely alter the registers in the deferred
624 * callback.
625 *
626 * In the scenario that we go from a valid to invalid
627 * and then back to valid FBC configuration we have
628 * no strict enforcement that a vblank occurred since
629 * disabling the FBC. However, along all current pipe
630 * disabling paths we do need to wait for a vblank at
631 * some point. And we wait before enabling FBC anyway.
632 */
633 DRM_DEBUG_KMS("disabling active FBC for update\n");
634 intel_disable_fbc(dev);
635 }
636
993495ae 637 intel_enable_fbc(crtc);
29ebf90f 638 dev_priv->fbc.no_fbc_reason = FBC_OK;
85208be0
ED		 639 return;
640
641out_disable:
642 /* Multiple disables should be harmless */
643 if (intel_fbc_enabled(dev)) {
644 DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
645 intel_disable_fbc(dev);
646 }
11be49eb 647 i915_gem_stolen_cleanup_compression(dev);
85208be0
ED		 648}
649
c921aba8
DV		 650static void i915_pineview_get_mem_freq(struct drm_device *dev)
651{
50227e1c 652 struct drm_i915_private *dev_priv = dev->dev_private;
c921aba8
DV		 653 u32 tmp;
654
655 tmp = I915_READ(CLKCFG);
656
657 switch (tmp & CLKCFG_FSB_MASK) {
658 case CLKCFG_FSB_533:
659 dev_priv->fsb_freq = 533; /* 133*4 */
660 break;
661 case CLKCFG_FSB_800:
662 dev_priv->fsb_freq = 800; /* 200*4 */
663 break;
664 case CLKCFG_FSB_667:
665 dev_priv->fsb_freq = 667; /* 167*4 */
666 break;
667 case CLKCFG_FSB_400:
668 dev_priv->fsb_freq = 400; /* 100*4 */
669 break;
670 }
671
672 switch (tmp & CLKCFG_MEM_MASK) {
673 case CLKCFG_MEM_533:
674 dev_priv->mem_freq = 533;
675 break;
676 case CLKCFG_MEM_667:
677 dev_priv->mem_freq = 667;
678 break;
679 case CLKCFG_MEM_800:
680 dev_priv->mem_freq = 800;
681 break;
682 }
683
684 /* detect pineview DDR3 setting */
685 tmp = I915_READ(CSHRDDR3CTL);
686 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
687}
688
689static void i915_ironlake_get_mem_freq(struct drm_device *dev)
690{
50227e1c 691 struct drm_i915_private *dev_priv = dev->dev_private;
c921aba8
DV		 692 u16 ddrpll, csipll;
693
694 ddrpll = I915_READ16(DDRMPLL1);
695 csipll = I915_READ16(CSIPLL0);
696
697 switch (ddrpll & 0xff) {
698 case 0xc:
699 dev_priv->mem_freq = 800;
700 break;
701 case 0x10:
702 dev_priv->mem_freq = 1066;
703 break;
704 case 0x14:
705 dev_priv->mem_freq = 1333;
706 break;
707 case 0x18:
708 dev_priv->mem_freq = 1600;
709 break;
710 default:
711 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
712 ddrpll & 0xff);
713 dev_priv->mem_freq = 0;
714 break;
715 }
716
20e4d407 717 dev_priv->ips.r_t = dev_priv->mem_freq;
c921aba8
DV		 718
719 switch (csipll & 0x3ff) {
720 case 0x00c:
721 dev_priv->fsb_freq = 3200;
722 break;
723 case 0x00e:
724 dev_priv->fsb_freq = 3733;
725 break;
726 case 0x010:
727 dev_priv->fsb_freq = 4266;
728 break;
729 case 0x012:
730 dev_priv->fsb_freq = 4800;
731 break;
732 case 0x014:
733 dev_priv->fsb_freq = 5333;
734 break;
735 case 0x016:
736 dev_priv->fsb_freq = 5866;
737 break;
738 case 0x018:
739 dev_priv->fsb_freq = 6400;
740 break;
741 default:
742 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
743 csipll & 0x3ff);
744 dev_priv->fsb_freq = 0;
745 break;
746 }
747
748 if (dev_priv->fsb_freq == 3200) {
20e4d407 749 dev_priv->ips.c_m = 0;
c921aba8 750 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
20e4d407 751 dev_priv->ips.c_m = 1;
c921aba8 752 } else {
20e4d407 753 dev_priv->ips.c_m = 2;
c921aba8
DV		 754 }
755}
756
b445e3b0
ED		 757static const struct cxsr_latency cxsr_latency_table[] = {
758 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
759 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
760 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
761 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
762 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
763
764 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
765 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
766 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
767 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
768 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
769
770 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
771 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
772 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
773 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
774 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
775
776 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
777 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
778 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
779 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
780 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
781
782 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
783 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
784 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
785 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
786 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
787
788 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
789 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
790 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
791 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
792 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
793};
794
63c62275 795static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
b445e3b0
ED		 796 int is_ddr3,
797 int fsb,
798 int mem)
799{
800 const struct cxsr_latency *latency;
801 int i;
802
803 if (fsb == 0 || mem == 0)
804 return NULL;
805
806 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
807 latency = &cxsr_latency_table[i];
808 if (is_desktop == latency->is_desktop &&
809 is_ddr3 == latency->is_ddr3 &&
810 fsb == latency->fsb_freq && mem == latency->mem_freq)
811 return latency;
812 }
813
814 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
815
816 return NULL;
817}
818
5209b1f4 819void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
b445e3b0 820{
5209b1f4
ID		 821 struct drm_device *dev = dev_priv->dev;
822 u32 val;
b445e3b0 823
5209b1f4
ID		 824 if (IS_VALLEYVIEW(dev)) {
825 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
826 } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) {
827 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
828 } else if (IS_PINEVIEW(dev)) {
829 val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
830 val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
831 I915_WRITE(DSPFW3, val);
832 } else if (IS_I945G(dev) || IS_I945GM(dev)) {
833 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
834 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
835 I915_WRITE(FW_BLC_SELF, val);
836 } else if (IS_I915GM(dev)) {
837 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
838 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
839 I915_WRITE(INSTPM, val);
840 } else {
841 return;
842 }
843
844 DRM_DEBUG_KMS("memory self-refresh is %s\n",
845 enable ? "enabled" : "disabled");
b445e3b0
ED		 846}
847
848/*
849 * Latency for FIFO fetches is dependent on several factors:
850 * - memory configuration (speed, channels)
851 * - chipset
852 * - current MCH state
853 * It can be fairly high in some situations, so here we assume a fairly
854 * pessimal value. It's a tradeoff between extra memory fetches (if we
855 * set this value too high, the FIFO will fetch frequently to stay full)
856 * and power consumption (set it too low to save power and we might see
857 * FIFO underruns and display "flicker").
858 *
859 * A value of 5us seems to be a good balance; safe for very low end
860 * platforms but not overly aggressive on lower latency configs.
861 */
862static const int latency_ns = 5000;
863
1fa61106 864static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
b445e3b0
ED		 865{
866 struct drm_i915_private *dev_priv = dev->dev_private;
867 uint32_t dsparb = I915_READ(DSPARB);
868 int size;
869
870 size = dsparb & 0x7f;
871 if (plane)
872 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
873
874 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
875 plane ? "B" : "A", size);
876
877 return size;
878}
879
feb56b93 880static int i830_get_fifo_size(struct drm_device *dev, int plane)
b445e3b0
ED		 881{
882 struct drm_i915_private *dev_priv = dev->dev_private;
883 uint32_t dsparb = I915_READ(DSPARB);
884 int size;
885
886 size = dsparb & 0x1ff;
887 if (plane)
888 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
889 size >>= 1; /* Convert to cachelines */
890
891 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
892 plane ? "B" : "A", size);
893
894 return size;
895}
896
1fa61106 897static int i845_get_fifo_size(struct drm_device *dev, int plane)
b445e3b0
ED		 898{
899 struct drm_i915_private *dev_priv = dev->dev_private;
900 uint32_t dsparb = I915_READ(DSPARB);
901 int size;
902
903 size = dsparb & 0x7f;
904 size >>= 2; /* Convert to cachelines */
905
906 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
907 plane ? "B" : "A",
908 size);
909
910 return size;
911}
912
b445e3b0
ED		 913/* Pineview has different values for various configs */
914static const struct intel_watermark_params pineview_display_wm = {
e0f0273e
VS		 915 .fifo_size = PINEVIEW_DISPLAY_FIFO,
916 .max_wm = PINEVIEW_MAX_WM,
917 .default_wm = PINEVIEW_DFT_WM,
918 .guard_size = PINEVIEW_GUARD_WM,
919 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
b445e3b0
ED		 920};
921static const struct intel_watermark_params pineview_display_hplloff_wm = {
e0f0273e
VS		 922 .fifo_size = PINEVIEW_DISPLAY_FIFO,
923 .max_wm = PINEVIEW_MAX_WM,
924 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
925 .guard_size = PINEVIEW_GUARD_WM,
926 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
b445e3b0
ED		 927};
928static const struct intel_watermark_params pineview_cursor_wm = {
e0f0273e
VS		 929 .fifo_size = PINEVIEW_CURSOR_FIFO,
930 .max_wm = PINEVIEW_CURSOR_MAX_WM,
931 .default_wm = PINEVIEW_CURSOR_DFT_WM,
932 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
933 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
b445e3b0
ED		 934};
935static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
e0f0273e
VS		 936 .fifo_size = PINEVIEW_CURSOR_FIFO,
937 .max_wm = PINEVIEW_CURSOR_MAX_WM,
938 .default_wm = PINEVIEW_CURSOR_DFT_WM,
939 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
940 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
b445e3b0
ED		 941};
942static const struct intel_watermark_params g4x_wm_info = {
e0f0273e
VS		 943 .fifo_size = G4X_FIFO_SIZE,
944 .max_wm = G4X_MAX_WM,
945 .default_wm = G4X_MAX_WM,
946 .guard_size = 2,
947 .cacheline_size = G4X_FIFO_LINE_SIZE,
b445e3b0
ED		 948};
949static const struct intel_watermark_params g4x_cursor_wm_info = {
e0f0273e
VS		 950 .fifo_size = I965_CURSOR_FIFO,
951 .max_wm = I965_CURSOR_MAX_WM,
952 .default_wm = I965_CURSOR_DFT_WM,
953 .guard_size = 2,
954 .cacheline_size = G4X_FIFO_LINE_SIZE,
b445e3b0
ED		 955};
956static const struct intel_watermark_params valleyview_wm_info = {
e0f0273e
VS		 957 .fifo_size = VALLEYVIEW_FIFO_SIZE,
958 .max_wm = VALLEYVIEW_MAX_WM,
959 .default_wm = VALLEYVIEW_MAX_WM,
960 .guard_size = 2,
961 .cacheline_size = G4X_FIFO_LINE_SIZE,
b445e3b0
ED		 962};
963static const struct intel_watermark_params valleyview_cursor_wm_info = {
e0f0273e
VS		 964 .fifo_size = I965_CURSOR_FIFO,
965 .max_wm = VALLEYVIEW_CURSOR_MAX_WM,
966 .default_wm = I965_CURSOR_DFT_WM,
967 .guard_size = 2,
968 .cacheline_size = G4X_FIFO_LINE_SIZE,
b445e3b0
ED		 969};
970static const struct intel_watermark_params i965_cursor_wm_info = {
e0f0273e
VS		 971 .fifo_size = I965_CURSOR_FIFO,
972 .max_wm = I965_CURSOR_MAX_WM,
973 .default_wm = I965_CURSOR_DFT_WM,
974 .guard_size = 2,
975 .cacheline_size = I915_FIFO_LINE_SIZE,
b445e3b0
ED		 976};
977static const struct intel_watermark_params i945_wm_info = {
e0f0273e
VS		 978 .fifo_size = I945_FIFO_SIZE,
979 .max_wm = I915_MAX_WM,
980 .default_wm = 1,
981 .guard_size = 2,
982 .cacheline_size = I915_FIFO_LINE_SIZE,
b445e3b0
ED		 983};
984static const struct intel_watermark_params i915_wm_info = {
e0f0273e
VS		 985 .fifo_size = I915_FIFO_SIZE,
986 .max_wm = I915_MAX_WM,
987 .default_wm = 1,
988 .guard_size = 2,
989 .cacheline_size = I915_FIFO_LINE_SIZE,
b445e3b0 990};
feb56b93 991static const struct intel_watermark_params i830_wm_info = {
e0f0273e
VS		 992 .fifo_size = I855GM_FIFO_SIZE,
993 .max_wm = I915_MAX_WM,
994 .default_wm = 1,
995 .guard_size = 2,
996 .cacheline_size = I830_FIFO_LINE_SIZE,
b445e3b0 997};
feb56b93 998static const struct intel_watermark_params i845_wm_info = {
e0f0273e
VS		 999 .fifo_size = I830_FIFO_SIZE,
1000 .max_wm = I915_MAX_WM,
1001 .default_wm = 1,
1002 .guard_size = 2,
1003 .cacheline_size = I830_FIFO_LINE_SIZE,
b445e3b0
ED		 1004};
1005
b445e3b0
ED		 1006/**
1007 * intel_calculate_wm - calculate watermark level
1008 * @clock_in_khz: pixel clock
1009 * @wm: chip FIFO params
1010 * @pixel_size: display pixel size
1011 * @latency_ns: memory latency for the platform
1012 *
1013 * Calculate the watermark level (the level at which the display plane will
1014 * start fetching from memory again). Each chip has a different display
1015 * FIFO size and allocation, so the caller needs to figure that out and pass
1016 * in the correct intel_watermark_params structure.
1017 *
1018 * As the pixel clock runs, the FIFO will be drained at a rate that depends
1019 * on the pixel size. When it reaches the watermark level, it'll start
1020 * fetching FIFO line sized based chunks from memory until the FIFO fills
1021 * past the watermark point. If the FIFO drains completely, a FIFO underrun
1022 * will occur, and a display engine hang could result.
1023 */
1024static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
1025 const struct intel_watermark_params *wm,
1026 int fifo_size,
1027 int pixel_size,
1028 unsigned long latency_ns)
1029{
1030 long entries_required, wm_size;
1031
1032 /*
1033 * Note: we need to make sure we don't overflow for various clock &
1034 * latency values.
1035 * clocks go from a few thousand to several hundred thousand.
1036 * latency is usually a few thousand
1037 */
1038 entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
1039 1000;
1040 entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
1041
1042 DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
1043
1044 wm_size = fifo_size - (entries_required + wm->guard_size);
1045
1046 DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
1047
1048 /* Don't promote wm_size to unsigned... */
1049 if (wm_size > (long)wm->max_wm)
1050 wm_size = wm->max_wm;
1051 if (wm_size <= 0)
1052 wm_size = wm->default_wm;
1053 return wm_size;
1054}
1055
1056static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
1057{
1058 struct drm_crtc *crtc, *enabled = NULL;
1059
70e1e0ec 1060 for_each_crtc(dev, crtc) {
3490ea5d 1061 if (intel_crtc_active(crtc)) {
b445e3b0
ED		 1062 if (enabled)
1063 return NULL;
1064 enabled = crtc;
1065 }
1066 }
1067
1068 return enabled;
1069}
1070
46ba614c 1071static void pineview_update_wm(struct drm_crtc *unused_crtc)
b445e3b0 1072{
46ba614c 1073 struct drm_device *dev = unused_crtc->dev;
b445e3b0
ED		 1074 struct drm_i915_private *dev_priv = dev->dev_private;
1075 struct drm_crtc *crtc;
1076 const struct cxsr_latency *latency;
1077 u32 reg;
1078 unsigned long wm;
1079
1080 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
1081 dev_priv->fsb_freq, dev_priv->mem_freq);
1082 if (!latency) {
1083 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
5209b1f4 1084 intel_set_memory_cxsr(dev_priv, false);
b445e3b0
ED		 1085 return;
1086 }
1087
1088 crtc = single_enabled_crtc(dev);
1089 if (crtc) {
241bfc38 1090 const struct drm_display_mode *adjusted_mode;
f4510a27 1091 int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
241bfc38
DL		 1092 int clock;
1093
1094 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1095 clock = adjusted_mode->crtc_clock;
b445e3b0
ED		 1096
1097 /* Display SR */
1098 wm = intel_calculate_wm(clock, &pineview_display_wm,
1099 pineview_display_wm.fifo_size,
1100 pixel_size, latency->display_sr);
1101 reg = I915_READ(DSPFW1);
1102 reg &= ~DSPFW_SR_MASK;
1103 reg |= wm << DSPFW_SR_SHIFT;
1104 I915_WRITE(DSPFW1, reg);
1105 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
1106
1107 /* cursor SR */
1108 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
1109 pineview_display_wm.fifo_size,
1110 pixel_size, latency->cursor_sr);
1111 reg = I915_READ(DSPFW3);
1112 reg &= ~DSPFW_CURSOR_SR_MASK;
1113 reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
1114 I915_WRITE(DSPFW3, reg);
1115
1116 /* Display HPLL off SR */
1117 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
1118 pineview_display_hplloff_wm.fifo_size,
1119 pixel_size, latency->display_hpll_disable);
1120 reg = I915_READ(DSPFW3);
1121 reg &= ~DSPFW_HPLL_SR_MASK;
1122 reg |= wm & DSPFW_HPLL_SR_MASK;
1123 I915_WRITE(DSPFW3, reg);
1124
1125 /* cursor HPLL off SR */
1126 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
1127 pineview_display_hplloff_wm.fifo_size,
1128 pixel_size, latency->cursor_hpll_disable);
1129 reg = I915_READ(DSPFW3);
1130 reg &= ~DSPFW_HPLL_CURSOR_MASK;
1131 reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
1132 I915_WRITE(DSPFW3, reg);
1133 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
1134
5209b1f4 1135 intel_set_memory_cxsr(dev_priv, true);
b445e3b0 1136 } else {
5209b1f4 1137 intel_set_memory_cxsr(dev_priv, false);
b445e3b0
ED		 1138 }
1139}
1140
1141static bool g4x_compute_wm0(struct drm_device *dev,
1142 int plane,
1143 const struct intel_watermark_params *display,
1144 int display_latency_ns,
1145 const struct intel_watermark_params *cursor,
1146 int cursor_latency_ns,
1147 int *plane_wm,
1148 int *cursor_wm)
1149{
1150 struct drm_crtc *crtc;
4fe8590a 1151 const struct drm_display_mode *adjusted_mode;
b445e3b0
ED		 1152 int htotal, hdisplay, clock, pixel_size;
1153 int line_time_us, line_count;
1154 int entries, tlb_miss;
1155
1156 crtc = intel_get_crtc_for_plane(dev, plane);
3490ea5d 1157 if (!intel_crtc_active(crtc)) {
b445e3b0
ED		 1158 *cursor_wm = cursor->guard_size;
1159 *plane_wm = display->guard_size;
1160 return false;
1161 }
1162
4fe8590a 1163 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
241bfc38 1164 clock = adjusted_mode->crtc_clock;
fec8cba3 1165 htotal = adjusted_mode->crtc_htotal;
37327abd 1166 hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
f4510a27 1167 pixel_size = crtc->primary->fb->bits_per_pixel / 8;
b445e3b0
ED		 1168
1169 /* Use the small buffer method to calculate plane watermark */
1170 entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
1171 tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
1172 if (tlb_miss > 0)
1173 entries += tlb_miss;
1174 entries = DIV_ROUND_UP(entries, display->cacheline_size);
1175 *plane_wm = entries + display->guard_size;
1176 if (*plane_wm > (int)display->max_wm)
1177 *plane_wm = display->max_wm;
1178
1179 /* Use the large buffer method to calculate cursor watermark */
922044c9 1180 line_time_us = max(htotal * 1000 / clock, 1);
b445e3b0 1181 line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
7bb836dd 1182 entries = line_count * to_intel_crtc(crtc)->cursor_width * pixel_size;
b445e3b0
ED		 1183 tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
1184 if (tlb_miss > 0)
1185 entries += tlb_miss;
1186 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
1187 *cursor_wm = entries + cursor->guard_size;
1188 if (*cursor_wm > (int)cursor->max_wm)
1189 *cursor_wm = (int)cursor->max_wm;
1190
1191 return true;
1192}
1193
1194/*
1195 * Check the wm result.
1196 *
1197 * If any calculated watermark values is larger than the maximum value that
1198 * can be programmed into the associated watermark register, that watermark
1199 * must be disabled.
1200 */
1201static bool g4x_check_srwm(struct drm_device *dev,
1202 int display_wm, int cursor_wm,
1203 const struct intel_watermark_params *display,
1204 const struct intel_watermark_params *cursor)
1205{
1206 DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
1207 display_wm, cursor_wm);
1208
1209 if (display_wm > display->max_wm) {
1210 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
1211 display_wm, display->max_wm);
1212 return false;
1213 }
1214
1215 if (cursor_wm > cursor->max_wm) {
1216 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
1217 cursor_wm, cursor->max_wm);
1218 return false;
1219 }
1220
1221 if (!(display_wm || cursor_wm)) {
1222 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
1223 return false;
1224 }
1225
1226 return true;
1227}
1228
1229static bool g4x_compute_srwm(struct drm_device *dev,
1230 int plane,
1231 int latency_ns,
1232 const struct intel_watermark_params *display,
1233 const struct intel_watermark_params *cursor,
1234 int *display_wm, int *cursor_wm)
1235{
1236 struct drm_crtc *crtc;
4fe8590a 1237 const struct drm_display_mode *adjusted_mode;
b445e3b0
ED		 1238 int hdisplay, htotal, pixel_size, clock;
1239 unsigned long line_time_us;
1240 int line_count, line_size;
1241 int small, large;
1242 int entries;
1243
1244 if (!latency_ns) {
1245 *display_wm = *cursor_wm = 0;
1246 return false;
1247 }
1248
1249 crtc = intel_get_crtc_for_plane(dev, plane);
4fe8590a 1250 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
241bfc38 1251 clock = adjusted_mode->crtc_clock;
fec8cba3 1252 htotal = adjusted_mode->crtc_htotal;
37327abd 1253 hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
f4510a27 1254 pixel_size = crtc->primary->fb->bits_per_pixel / 8;
b445e3b0 1255
922044c9 1256 line_time_us = max(htotal * 1000 / clock, 1);
b445e3b0
ED		 1257 line_count = (latency_ns / line_time_us + 1000) / 1000;
1258 line_size = hdisplay * pixel_size;
1259
1260 /* Use the minimum of the small and large buffer method for primary */
1261 small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
1262 large = line_count * line_size;
1263
1264 entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
1265 *display_wm = entries + display->guard_size;
1266
1267 /* calculate the self-refresh watermark for display cursor */
7bb836dd 1268 entries = line_count * pixel_size * to_intel_crtc(crtc)->cursor_width;
b445e3b0
ED		 1269 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
1270 *cursor_wm = entries + cursor->guard_size;
1271
1272 return g4x_check_srwm(dev,
1273 *display_wm, *cursor_wm,
1274 display, cursor);
1275}
1276
1277static bool vlv_compute_drain_latency(struct drm_device *dev,
1278 int plane,
1279 int *plane_prec_mult,
1280 int *plane_dl,
1281 int *cursor_prec_mult,
1282 int *cursor_dl)
1283{
1284 struct drm_crtc *crtc;
1285 int clock, pixel_size;
1286 int entries;
1287
1288 crtc = intel_get_crtc_for_plane(dev, plane);
3490ea5d 1289 if (!intel_crtc_active(crtc))
b445e3b0
ED		 1290 return false;
1291
241bfc38 1292 clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
f4510a27 1293 pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */
b445e3b0
ED		 1294
1295 entries = (clock / 1000) * pixel_size;
1296 *plane_prec_mult = (entries > 256) ?
1297 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
1298 *plane_dl = (64 * (*plane_prec_mult) * 4) / ((clock / 1000) *
1299 pixel_size);
1300
1301 entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
1302 *cursor_prec_mult = (entries > 256) ?
1303 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
1304 *cursor_dl = (64 * (*cursor_prec_mult) * 4) / ((clock / 1000) * 4);
1305
1306 return true;
1307}
1308
1309/*
1310 * Update drain latency registers of memory arbiter
1311 *
1312 * Valleyview SoC has a new memory arbiter and needs drain latency registers
1313 * to be programmed. Each plane has a drain latency multiplier and a drain
1314 * latency value.
1315 */
1316
1317static void vlv_update_drain_latency(struct drm_device *dev)
1318{
1319 struct drm_i915_private *dev_priv = dev->dev_private;
1320 int planea_prec, planea_dl, planeb_prec, planeb_dl;
1321 int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
1322 int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
1323 either 16 or 32 */
1324
1325 /* For plane A, Cursor A */
1326 if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
1327 &cursor_prec_mult, &cursora_dl)) {
1328 cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1329 DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
1330 planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1331 DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
1332
1333 I915_WRITE(VLV_DDL1, cursora_prec |
1334 (cursora_dl << DDL_CURSORA_SHIFT) |
1335 planea_prec | planea_dl);
1336 }
1337
1338 /* For plane B, Cursor B */
1339 if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
1340 &cursor_prec_mult, &cursorb_dl)) {
1341 cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1342 DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
1343 planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1344 DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
1345
1346 I915_WRITE(VLV_DDL2, cursorb_prec |
1347 (cursorb_dl << DDL_CURSORB_SHIFT) |
1348 planeb_prec | planeb_dl);
1349 }
1350}
1351
1352#define single_plane_enabled(mask) is_power_of_2(mask)
1353
46ba614c 1354static void valleyview_update_wm(struct drm_crtc *crtc)
b445e3b0 1355{
46ba614c 1356 struct drm_device *dev = crtc->dev;
b445e3b0
ED		 1357 static const int sr_latency_ns = 12000;
1358 struct drm_i915_private *dev_priv = dev->dev_private;
1359 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1360 int plane_sr, cursor_sr;
af6c4575 1361 int ignore_plane_sr, ignore_cursor_sr;
b445e3b0 1362 unsigned int enabled = 0;
9858425c 1363 bool cxsr_enabled;
b445e3b0
ED		 1364
1365 vlv_update_drain_latency(dev);
1366
51cea1f4 1367 if (g4x_compute_wm0(dev, PIPE_A,
b445e3b0
ED		 1368 &valleyview_wm_info, latency_ns,
1369 &valleyview_cursor_wm_info, latency_ns,
1370 &planea_wm, &cursora_wm))
51cea1f4 1371 enabled |= 1 << PIPE_A;
b445e3b0 1372
51cea1f4 1373 if (g4x_compute_wm0(dev, PIPE_B,
b445e3b0
ED		 1374 &valleyview_wm_info, latency_ns,
1375 &valleyview_cursor_wm_info, latency_ns,
1376 &planeb_wm, &cursorb_wm))
51cea1f4 1377 enabled |= 1 << PIPE_B;
b445e3b0 1378
b445e3b0
ED		 1379 if (single_plane_enabled(enabled) &&
1380 g4x_compute_srwm(dev, ffs(enabled) - 1,
1381 sr_latency_ns,
1382 &valleyview_wm_info,
1383 &valleyview_cursor_wm_info,
af6c4575
CW		 1384 &plane_sr, &ignore_cursor_sr) &&
1385 g4x_compute_srwm(dev, ffs(enabled) - 1,
1386 2*sr_latency_ns,
1387 &valleyview_wm_info,
1388 &valleyview_cursor_wm_info,
52bd02d8 1389 &ignore_plane_sr, &cursor_sr)) {
9858425c 1390 cxsr_enabled = true;
52bd02d8 1391 } else {
9858425c 1392 cxsr_enabled = false;
5209b1f4 1393 intel_set_memory_cxsr(dev_priv, false);
52bd02d8
CW		 1394 plane_sr = cursor_sr = 0;
1395 }
b445e3b0
ED		 1396
1397 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1398 planea_wm, cursora_wm,
1399 planeb_wm, cursorb_wm,
1400 plane_sr, cursor_sr);
1401
1402 I915_WRITE(DSPFW1,
1403 (plane_sr << DSPFW_SR_SHIFT) |
1404 (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1405 (planeb_wm << DSPFW_PLANEB_SHIFT) |
1406 planea_wm);
1407 I915_WRITE(DSPFW2,
8c919b28 1408 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
b445e3b0
ED		 1409 (cursora_wm << DSPFW_CURSORA_SHIFT));
1410 I915_WRITE(DSPFW3,
8c919b28
CW		 1411 (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
1412 (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
9858425c
ID		 1413
1414 if (cxsr_enabled)
1415 intel_set_memory_cxsr(dev_priv, true);
b445e3b0
ED		 1416}
1417
46ba614c 1418static void g4x_update_wm(struct drm_crtc *crtc)
b445e3b0 1419{
46ba614c 1420 struct drm_device *dev = crtc->dev;
b445e3b0
ED		 1421 static const int sr_latency_ns = 12000;
1422 struct drm_i915_private *dev_priv = dev->dev_private;
1423 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1424 int plane_sr, cursor_sr;
1425 unsigned int enabled = 0;
9858425c 1426 bool cxsr_enabled;
b445e3b0 1427
51cea1f4 1428 if (g4x_compute_wm0(dev, PIPE_A,
b445e3b0
ED		 1429 &g4x_wm_info, latency_ns,
1430 &g4x_cursor_wm_info, latency_ns,
1431 &planea_wm, &cursora_wm))
51cea1f4 1432 enabled |= 1 << PIPE_A;
b445e3b0 1433
51cea1f4 1434 if (g4x_compute_wm0(dev, PIPE_B,
b445e3b0
ED		 1435 &g4x_wm_info, latency_ns,
1436 &g4x_cursor_wm_info, latency_ns,
1437 &planeb_wm, &cursorb_wm))
51cea1f4 1438 enabled |= 1 << PIPE_B;
b445e3b0 1439
b445e3b0
ED		 1440 if (single_plane_enabled(enabled) &&
1441 g4x_compute_srwm(dev, ffs(enabled) - 1,
1442 sr_latency_ns,
1443 &g4x_wm_info,
1444 &g4x_cursor_wm_info,
52bd02d8 1445 &plane_sr, &cursor_sr)) {
9858425c 1446 cxsr_enabled = true;
52bd02d8 1447 } else {
9858425c 1448 cxsr_enabled = false;
5209b1f4 1449 intel_set_memory_cxsr(dev_priv, false);
52bd02d8
CW		 1450 plane_sr = cursor_sr = 0;
1451 }
b445e3b0
ED		 1452
1453 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1454 planea_wm, cursora_wm,
1455 planeb_wm, cursorb_wm,
1456 plane_sr, cursor_sr);
1457
1458 I915_WRITE(DSPFW1,
1459 (plane_sr << DSPFW_SR_SHIFT) |
1460 (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1461 (planeb_wm << DSPFW_PLANEB_SHIFT) |
1462 planea_wm);
1463 I915_WRITE(DSPFW2,
8c919b28 1464 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
b445e3b0
ED		 1465 (cursora_wm << DSPFW_CURSORA_SHIFT));
1466 /* HPLL off in SR has some issues on G4x... disable it */
1467 I915_WRITE(DSPFW3,
8c919b28 1468 (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
b445e3b0 1469 (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
9858425c
ID		 1470
1471 if (cxsr_enabled)
1472 intel_set_memory_cxsr(dev_priv, true);
b445e3b0
ED		 1473}
1474
46ba614c 1475static void i965_update_wm(struct drm_crtc *unused_crtc)
b445e3b0 1476{
46ba614c 1477 struct drm_device *dev = unused_crtc->dev;
b445e3b0
ED		 1478 struct drm_i915_private *dev_priv = dev->dev_private;
1479 struct drm_crtc *crtc;
1480 int srwm = 1;
1481 int cursor_sr = 16;
9858425c 1482 bool cxsr_enabled;
b445e3b0
ED		 1483
1484 /* Calc sr entries for one plane configs */
1485 crtc = single_enabled_crtc(dev);
1486 if (crtc) {
1487 /* self-refresh has much higher latency */
1488 static const int sr_latency_ns = 12000;
4fe8590a
VS		 1489 const struct drm_display_mode *adjusted_mode =
1490 &to_intel_crtc(crtc)->config.adjusted_mode;
241bfc38 1491 int clock = adjusted_mode->crtc_clock;
fec8cba3 1492 int htotal = adjusted_mode->crtc_htotal;
37327abd 1493 int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
f4510a27 1494 int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
b445e3b0
ED		 1495 unsigned long line_time_us;
1496 int entries;
1497
922044c9 1498 line_time_us = max(htotal * 1000 / clock, 1);
b445e3b0
ED		 1499
1500 /* Use ns/us then divide to preserve precision */
1501 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1502 pixel_size * hdisplay;
1503 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
1504 srwm = I965_FIFO_SIZE - entries;
1505 if (srwm < 0)
1506 srwm = 1;
1507 srwm &= 0x1ff;
1508 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1509 entries, srwm);
1510
1511 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
7bb836dd 1512 pixel_size * to_intel_crtc(crtc)->cursor_width;
b445e3b0
ED		 1513 entries = DIV_ROUND_UP(entries,
1514 i965_cursor_wm_info.cacheline_size);
1515 cursor_sr = i965_cursor_wm_info.fifo_size -
1516 (entries + i965_cursor_wm_info.guard_size);
1517
1518 if (cursor_sr > i965_cursor_wm_info.max_wm)
1519 cursor_sr = i965_cursor_wm_info.max_wm;
1520
1521 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1522 "cursor %d\n", srwm, cursor_sr);
1523
9858425c 1524 cxsr_enabled = true;
b445e3b0 1525 } else {
9858425c 1526 cxsr_enabled = false;
b445e3b0 1527 /* Turn off self refresh if both pipes are enabled */
5209b1f4 1528 intel_set_memory_cxsr(dev_priv, false);
b445e3b0
ED		 1529 }
1530
1531 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1532 srwm);
1533
1534 /* 965 has limitations... */
1535 I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
1536 (8 << 16) | (8 << 8) | (8 << 0));
1537 I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
1538 /* update cursor SR watermark */
1539 I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
9858425c
ID		 1540
1541 if (cxsr_enabled)
1542 intel_set_memory_cxsr(dev_priv, true);
b445e3b0
ED		 1543}
1544
46ba614c 1545static void i9xx_update_wm(struct drm_crtc *unused_crtc)
b445e3b0 1546{
46ba614c 1547 struct drm_device *dev = unused_crtc->dev;
b445e3b0
ED		 1548 struct drm_i915_private *dev_priv = dev->dev_private;
1549 const struct intel_watermark_params *wm_info;
1550 uint32_t fwater_lo;
1551 uint32_t fwater_hi;
1552 int cwm, srwm = 1;
1553 int fifo_size;
1554 int planea_wm, planeb_wm;
1555 struct drm_crtc *crtc, *enabled = NULL;
1556
1557 if (IS_I945GM(dev))
1558 wm_info = &i945_wm_info;
1559 else if (!IS_GEN2(dev))
1560 wm_info = &i915_wm_info;
1561 else
feb56b93 1562 wm_info = &i830_wm_info;
b445e3b0
ED		 1563
1564 fifo_size = dev_priv->display.get_fifo_size(dev, 0);
1565 crtc = intel_get_crtc_for_plane(dev, 0);
3490ea5d 1566 if (intel_crtc_active(crtc)) {
241bfc38 1567 const struct drm_display_mode *adjusted_mode;
f4510a27 1568 int cpp = crtc->primary->fb->bits_per_pixel / 8;
b9e0bda3
CW		 1569 if (IS_GEN2(dev))
1570 cpp = 4;
1571
241bfc38
DL		 1572 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1573 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
b9e0bda3 1574 wm_info, fifo_size, cpp,
b445e3b0
ED		 1575 latency_ns);
1576 enabled = crtc;
1577 } else
1578 planea_wm = fifo_size - wm_info->guard_size;
1579
1580 fifo_size = dev_priv->display.get_fifo_size(dev, 1);
1581 crtc = intel_get_crtc_for_plane(dev, 1);
3490ea5d 1582 if (intel_crtc_active(crtc)) {
241bfc38 1583 const struct drm_display_mode *adjusted_mode;
f4510a27 1584 int cpp = crtc->primary->fb->bits_per_pixel / 8;
b9e0bda3
CW		 1585 if (IS_GEN2(dev))
1586 cpp = 4;
1587
241bfc38
DL		 1588 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1589 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
b9e0bda3 1590 wm_info, fifo_size, cpp,
b445e3b0
ED		 1591 latency_ns);
1592 if (enabled == NULL)
1593 enabled = crtc;
1594 else
1595 enabled = NULL;
1596 } else
1597 planeb_wm = fifo_size - wm_info->guard_size;
1598
1599 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
1600
2ab1bc9d
DV		 1601 if (IS_I915GM(dev) && enabled) {
1602 struct intel_framebuffer *fb;
1603
1604 fb = to_intel_framebuffer(enabled->primary->fb);
1605
1606 /* self-refresh seems busted with untiled */
1607 if (fb->obj->tiling_mode == I915_TILING_NONE)
1608 enabled = NULL;
1609 }
1610
b445e3b0
ED		 1611 /*
1612 * Overlay gets an aggressive default since video jitter is bad.
1613 */
1614 cwm = 2;
1615
1616 /* Play safe and disable self-refresh before adjusting watermarks. */
5209b1f4 1617 intel_set_memory_cxsr(dev_priv, false);
b445e3b0
ED		 1618
1619 /* Calc sr entries for one plane configs */
1620 if (HAS_FW_BLC(dev) && enabled) {
1621 /* self-refresh has much higher latency */
1622 static const int sr_latency_ns = 6000;
4fe8590a
VS		 1623 const struct drm_display_mode *adjusted_mode =
1624 &to_intel_crtc(enabled)->config.adjusted_mode;
241bfc38 1625 int clock = adjusted_mode->crtc_clock;
fec8cba3 1626 int htotal = adjusted_mode->crtc_htotal;
f727b490 1627 int hdisplay = to_intel_crtc(enabled)->config.pipe_src_w;
f4510a27 1628 int pixel_size = enabled->primary->fb->bits_per_pixel / 8;
b445e3b0
ED		 1629 unsigned long line_time_us;
1630 int entries;
1631
922044c9 1632 line_time_us = max(htotal * 1000 / clock, 1);
b445e3b0
ED		 1633
1634 /* Use ns/us then divide to preserve precision */
1635 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1636 pixel_size * hdisplay;
1637 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
1638 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
1639 srwm = wm_info->fifo_size - entries;
1640 if (srwm < 0)
1641 srwm = 1;
1642
1643 if (IS_I945G(dev) || IS_I945GM(dev))
1644 I915_WRITE(FW_BLC_SELF,
1645 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
1646 else if (IS_I915GM(dev))
1647 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
1648 }
1649
1650 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1651 planea_wm, planeb_wm, cwm, srwm);
1652
1653 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
1654 fwater_hi = (cwm & 0x1f);
1655
1656 /* Set request length to 8 cachelines per fetch */
1657 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
1658 fwater_hi = fwater_hi | (1 << 8);
1659
1660 I915_WRITE(FW_BLC, fwater_lo);
1661 I915_WRITE(FW_BLC2, fwater_hi);
1662
5209b1f4
ID		 1663 if (enabled)
1664 intel_set_memory_cxsr(dev_priv, true);
b445e3b0
ED		 1665}
1666
feb56b93 1667static void i845_update_wm(struct drm_crtc *unused_crtc)
b445e3b0 1668{
46ba614c 1669 struct drm_device *dev = unused_crtc->dev;
b445e3b0
ED		 1670 struct drm_i915_private *dev_priv = dev->dev_private;
1671 struct drm_crtc *crtc;
241bfc38 1672 const struct drm_display_mode *adjusted_mode;
b445e3b0
ED		 1673 uint32_t fwater_lo;
1674 int planea_wm;
1675
1676 crtc = single_enabled_crtc(dev);
1677 if (crtc == NULL)
1678 return;
1679
241bfc38
DL		 1680 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1681 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
feb56b93 1682 &i845_wm_info,
b445e3b0 1683 dev_priv->display.get_fifo_size(dev, 0),
b9e0bda3 1684 4, latency_ns);
b445e3b0
ED		 1685 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
1686 fwater_lo |= (3<<8) | planea_wm;
1687
1688 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
1689
1690 I915_WRITE(FW_BLC, fwater_lo);
1691}
1692
3658729a
VS		 1693static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
1694 struct drm_crtc *crtc)
801bcfff
PZ		 1695{
1696 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
fd4daa9c 1697 uint32_t pixel_rate;
801bcfff 1698
241bfc38 1699 pixel_rate = intel_crtc->config.adjusted_mode.crtc_clock;
801bcfff
PZ		 1700
1701 /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
1702 * adjust the pixel_rate here. */
1703
fd4daa9c 1704 if (intel_crtc->config.pch_pfit.enabled) {
801bcfff 1705 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
fd4daa9c 1706 uint32_t pfit_size = intel_crtc->config.pch_pfit.size;
801bcfff 1707
37327abd
VS		 1708 pipe_w = intel_crtc->config.pipe_src_w;
1709 pipe_h = intel_crtc->config.pipe_src_h;
801bcfff
PZ		 1710 pfit_w = (pfit_size >> 16) & 0xFFFF;
1711 pfit_h = pfit_size & 0xFFFF;
1712 if (pipe_w < pfit_w)
1713 pipe_w = pfit_w;
1714 if (pipe_h < pfit_h)
1715 pipe_h = pfit_h;
1716
1717 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
1718 pfit_w * pfit_h);
1719 }
1720
1721 return pixel_rate;
1722}
1723
37126462 1724/* latency must be in 0.1us units. */
23297044 1725static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
801bcfff
PZ		 1726 uint32_t latency)
1727{
1728 uint64_t ret;
1729
3312ba65
VS		 1730 if (WARN(latency == 0, "Latency value missing\n"))
1731 return UINT_MAX;
1732
801bcfff
PZ		 1733 ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
1734 ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
1735
1736 return ret;
1737}
1738
37126462 1739/* latency must be in 0.1us units. */
23297044 1740static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
801bcfff
PZ		 1741 uint32_t horiz_pixels, uint8_t bytes_per_pixel,
1742 uint32_t latency)
1743{
1744 uint32_t ret;
1745
3312ba65
VS		 1746 if (WARN(latency == 0, "Latency value missing\n"))
1747 return UINT_MAX;
1748
801bcfff
PZ		 1749 ret = (latency * pixel_rate) / (pipe_htotal * 10000);
1750 ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
1751 ret = DIV_ROUND_UP(ret, 64) + 2;
1752 return ret;
1753}
1754
23297044 1755static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
cca32e9a
PZ		 1756 uint8_t bytes_per_pixel)
1757{
1758 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
1759}
1760
820c1980 1761struct ilk_pipe_wm_parameters {
801bcfff 1762 bool active;
801bcfff
PZ		 1763 uint32_t pipe_htotal;
1764 uint32_t pixel_rate;
c35426d2
VS		 1765 struct intel_plane_wm_parameters pri;
1766 struct intel_plane_wm_parameters spr;
1767 struct intel_plane_wm_parameters cur;
801bcfff
PZ		 1768};
1769
820c1980 1770struct ilk_wm_maximums {
cca32e9a
PZ		 1771 uint16_t pri;
1772 uint16_t spr;
1773 uint16_t cur;
1774 uint16_t fbc;
1775};
1776
240264f4
VS		 1777/* used in computing the new watermarks state */
1778struct intel_wm_config {
1779 unsigned int num_pipes_active;
1780 bool sprites_enabled;
1781 bool sprites_scaled;
240264f4
VS		 1782};
1783
37126462
VS		 1784/*
1785 * For both WM_PIPE and WM_LP.
1786 * mem_value must be in 0.1us units.
1787 */
820c1980 1788static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
cca32e9a
PZ		 1789 uint32_t mem_value,
1790 bool is_lp)
801bcfff 1791{
cca32e9a
PZ		 1792 uint32_t method1, method2;
1793
c35426d2 1794 if (!params->active || !params->pri.enabled)
801bcfff
PZ		 1795 return 0;
1796
23297044 1797 method1 = ilk_wm_method1(params->pixel_rate,
c35426d2 1798 params->pri.bytes_per_pixel,
cca32e9a
PZ		 1799 mem_value);
1800
1801 if (!is_lp)
1802 return method1;
1803
23297044 1804 method2 = ilk_wm_method2(params->pixel_rate,
cca32e9a 1805 params->pipe_htotal,
c35426d2
VS		 1806 params->pri.horiz_pixels,
1807 params->pri.bytes_per_pixel,
cca32e9a
PZ		 1808 mem_value);
1809
1810 return min(method1, method2);
801bcfff
PZ		 1811}
1812
37126462
VS		 1813/*
1814 * For both WM_PIPE and WM_LP.
1815 * mem_value must be in 0.1us units.
1816 */
820c1980 1817static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
801bcfff
PZ		 1818 uint32_t mem_value)
1819{
1820 uint32_t method1, method2;
1821
c35426d2 1822 if (!params->active || !params->spr.enabled)
801bcfff
PZ		 1823 return 0;
1824
23297044 1825 method1 = ilk_wm_method1(params->pixel_rate,
c35426d2 1826 params->spr.bytes_per_pixel,
801bcfff 1827 mem_value);
23297044 1828 method2 = ilk_wm_method2(params->pixel_rate,
801bcfff 1829 params->pipe_htotal,
c35426d2
VS		 1830 params->spr.horiz_pixels,
1831 params->spr.bytes_per_pixel,
801bcfff
PZ		 1832 mem_value);
1833 return min(method1, method2);
1834}
1835
37126462
VS		 1836/*
1837 * For both WM_PIPE and WM_LP.
1838 * mem_value must be in 0.1us units.
1839 */
820c1980 1840static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
801bcfff
PZ		 1841 uint32_t mem_value)
1842{
c35426d2 1843 if (!params->active || !params->cur.enabled)
801bcfff
PZ		 1844 return 0;
1845
23297044 1846 return ilk_wm_method2(params->pixel_rate,
801bcfff 1847 params->pipe_htotal,
c35426d2
VS		 1848 params->cur.horiz_pixels,
1849 params->cur.bytes_per_pixel,
801bcfff
PZ		 1850 mem_value);
1851}
1852
cca32e9a 1853/* Only for WM_LP. */
820c1980 1854static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
1fda9882 1855 uint32_t pri_val)
cca32e9a 1856{
c35426d2 1857 if (!params->active || !params->pri.enabled)
cca32e9a
PZ		 1858 return 0;
1859
23297044 1860 return ilk_wm_fbc(pri_val,
c35426d2
VS		 1861 params->pri.horiz_pixels,
1862 params->pri.bytes_per_pixel);
cca32e9a
PZ		 1863}
1864
158ae64f
VS		 1865static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
1866{
416f4727
VS		 1867 if (INTEL_INFO(dev)->gen >= 8)
1868 return 3072;
1869 else if (INTEL_INFO(dev)->gen >= 7)
158ae64f
VS		 1870 return 768;
1871 else
1872 return 512;
1873}
1874
4e975081
VS		 1875static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
1876 int level, bool is_sprite)
1877{
1878 if (INTEL_INFO(dev)->gen >= 8)
1879 /* BDW primary/sprite plane watermarks */
1880 return level == 0 ? 255 : 2047;
1881 else if (INTEL_INFO(dev)->gen >= 7)
1882 /* IVB/HSW primary/sprite plane watermarks */
1883 return level == 0 ? 127 : 1023;
1884 else if (!is_sprite)
1885 /* ILK/SNB primary plane watermarks */
1886 return level == 0 ? 127 : 511;
1887 else
1888 /* ILK/SNB sprite plane watermarks */
1889 return level == 0 ? 63 : 255;
1890}
1891
1892static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
1893 int level)
1894{
1895 if (INTEL_INFO(dev)->gen >= 7)
1896 return level == 0 ? 63 : 255;
1897 else
1898 return level == 0 ? 31 : 63;
1899}
1900
1901static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
1902{
1903 if (INTEL_INFO(dev)->gen >= 8)
1904 return 31;
1905 else
1906 return 15;
1907}
1908
158ae64f
VS		 1909/* Calculate the maximum primary/sprite plane watermark */
1910static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
1911 int level,
240264f4 1912 const struct intel_wm_config *config,
158ae64f
VS		 1913 enum intel_ddb_partitioning ddb_partitioning,
1914 bool is_sprite)
1915{
1916 unsigned int fifo_size = ilk_display_fifo_size(dev);
158ae64f
VS		 1917
1918 /* if sprites aren't enabled, sprites get nothing */
240264f4 1919 if (is_sprite && !config->sprites_enabled)
158ae64f
VS		 1920 return 0;
1921
1922 /* HSW allows LP1+ watermarks even with multiple pipes */
240264f4 1923 if (level == 0 || config->num_pipes_active > 1) {
158ae64f
VS		 1924 fifo_size /= INTEL_INFO(dev)->num_pipes;
1925
1926 /*
1927 * For some reason the non self refresh
1928 * FIFO size is only half of the self
1929 * refresh FIFO size on ILK/SNB.
1930 */
1931 if (INTEL_INFO(dev)->gen <= 6)
1932 fifo_size /= 2;
1933 }
1934
240264f4 1935 if (config->sprites_enabled) {
158ae64f
VS		 1936 /* level 0 is always calculated with 1:1 split */
1937 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
1938 if (is_sprite)
1939 fifo_size *= 5;
1940 fifo_size /= 6;
1941 } else {
1942 fifo_size /= 2;
1943 }
1944 }
1945
1946 /* clamp to max that the registers can hold */
4e975081 1947 return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
158ae64f
VS		 1948}
1949
1950/* Calculate the maximum cursor plane watermark */
1951static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
240264f4
VS		 1952 int level,
1953 const struct intel_wm_config *config)
158ae64f
VS		 1954{
1955 /* HSW LP1+ watermarks w/ multiple pipes */
240264f4 1956 if (level > 0 && config->num_pipes_active > 1)
158ae64f
VS		 1957 return 64;
1958
1959 /* otherwise just report max that registers can hold */
4e975081 1960 return ilk_cursor_wm_reg_max(dev, level);
158ae64f
VS		 1961}
1962
d34ff9c6 1963static void ilk_compute_wm_maximums(const struct drm_device *dev,
34982fe1
VS		 1964 int level,
1965 const struct intel_wm_config *config,
1966 enum intel_ddb_partitioning ddb_partitioning,
820c1980 1967 struct ilk_wm_maximums *max)
158ae64f 1968{
240264f4
VS		 1969 max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
1970 max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
1971 max->cur = ilk_cursor_wm_max(dev, level, config);
4e975081 1972 max->fbc = ilk_fbc_wm_reg_max(dev);
158ae64f
VS		 1973}
1974
a3cb4048
VS		 1975static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
1976 int level,
1977 struct ilk_wm_maximums *max)
1978{
1979 max->pri = ilk_plane_wm_reg_max(dev, level, false);
1980 max->spr = ilk_plane_wm_reg_max(dev, level, true);
1981 max->cur = ilk_cursor_wm_reg_max(dev, level);
1982 max->fbc = ilk_fbc_wm_reg_max(dev);
1983}
1984
d9395655 1985static bool ilk_validate_wm_level(int level,
820c1980 1986 const struct ilk_wm_maximums *max,
d9395655 1987 struct intel_wm_level *result)
a9786a11
VS		 1988{
1989 bool ret;
1990
1991 /* already determined to be invalid? */
1992 if (!result->enable)
1993 return false;
1994
1995 result->enable = result->pri_val <= max->pri &&
1996 result->spr_val <= max->spr &&
1997 result->cur_val <= max->cur;
1998
1999 ret = result->enable;
2000
2001 /*
2002 * HACK until we can pre-compute everything,
2003 * and thus fail gracefully if LP0 watermarks
2004 * are exceeded...
2005 */
2006 if (level == 0 && !result->enable) {
2007 if (result->pri_val > max->pri)
2008 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2009 level, result->pri_val, max->pri);
2010 if (result->spr_val > max->spr)
2011 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2012 level, result->spr_val, max->spr);
2013 if (result->cur_val > max->cur)
2014 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2015 level, result->cur_val, max->cur);
2016
2017 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
2018 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
2019 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
2020 result->enable = true;
2021 }
2022
a9786a11
VS		 2023 return ret;
2024}
2025
d34ff9c6 2026static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
6f5ddd17 2027 int level,
820c1980 2028 const struct ilk_pipe_wm_parameters *p,
1fd527cc 2029 struct intel_wm_level *result)
6f5ddd17
VS		 2030{
2031 uint16_t pri_latency = dev_priv->wm.pri_latency[level];
2032 uint16_t spr_latency = dev_priv->wm.spr_latency[level];
2033 uint16_t cur_latency = dev_priv->wm.cur_latency[level];
2034
2035 /* WM1+ latency values stored in 0.5us units */
2036 if (level > 0) {
2037 pri_latency *= 5;
2038 spr_latency *= 5;
2039 cur_latency *= 5;
2040 }
2041
2042 result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
2043 result->spr_val = ilk_compute_spr_wm(p, spr_latency);
2044 result->cur_val = ilk_compute_cur_wm(p, cur_latency);
2045 result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
2046 result->enable = true;
2047}
2048
801bcfff
PZ		 2049static uint32_t
2050hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc)
1f8eeabf
ED		 2051{
2052 struct drm_i915_private *dev_priv = dev->dev_private;
1011d8c4 2053 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1011d8c4 2054 struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
85a02deb 2055 u32 linetime, ips_linetime;
1f8eeabf 2056
801bcfff
PZ		 2057 if (!intel_crtc_active(crtc))
2058 return 0;
1011d8c4 2059
1f8eeabf
ED		 2060 /* The WM are computed with base on how long it takes to fill a single
2061 * row at the given clock rate, multiplied by 8.
2062 * */
fec8cba3
JB		 2063 linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
2064 mode->crtc_clock);
2065 ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
85a02deb 2066 intel_ddi_get_cdclk_freq(dev_priv));
1f8eeabf 2067
801bcfff
PZ		 2068 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2069 PIPE_WM_LINETIME_TIME(linetime);
1f8eeabf
ED		 2070}
2071
12b134df
VS		 2072static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[5])
2073{
2074 struct drm_i915_private *dev_priv = dev->dev_private;
2075
a42a5719 2076 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
12b134df
VS		 2077 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2078
2079 wm[0] = (sskpd >> 56) & 0xFF;
2080 if (wm[0] == 0)
2081 wm[0] = sskpd & 0xF;
e5d5019e
VS		 2082 wm[1] = (sskpd >> 4) & 0xFF;
2083 wm[2] = (sskpd >> 12) & 0xFF;
2084 wm[3] = (sskpd >> 20) & 0x1FF;
2085 wm[4] = (sskpd >> 32) & 0x1FF;
63cf9a13
VS		 2086 } else if (INTEL_INFO(dev)->gen >= 6) {
2087 uint32_t sskpd = I915_READ(MCH_SSKPD);
2088
2089 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2090 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2091 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2092 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
3a88d0ac
VS		 2093 } else if (INTEL_INFO(dev)->gen >= 5) {
2094 uint32_t mltr = I915_READ(MLTR_ILK);
2095
2096 /* ILK primary LP0 latency is 700 ns */
2097 wm[0] = 7;
2098 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2099 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
12b134df
VS		 2100 }
2101}
2102
53615a5e
VS		 2103static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
2104{
2105 /* ILK sprite LP0 latency is 1300 ns */
2106 if (INTEL_INFO(dev)->gen == 5)
2107 wm[0] = 13;
2108}
2109
2110static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
2111{
2112 /* ILK cursor LP0 latency is 1300 ns */
2113 if (INTEL_INFO(dev)->gen == 5)
2114 wm[0] = 13;
2115
2116 /* WaDoubleCursorLP3Latency:ivb */
2117 if (IS_IVYBRIDGE(dev))
2118 wm[3] *= 2;
2119}
2120
546c81fd 2121int ilk_wm_max_level(const struct drm_device *dev)
26ec971e 2122{
26ec971e 2123 /* how many WM levels are we expecting */
a42a5719 2124 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
ad0d6dc4 2125 return 4;
26ec971e 2126 else if (INTEL_INFO(dev)->gen >= 6)
ad0d6dc4 2127 return 3;
26ec971e 2128 else
ad0d6dc4
VS		 2129 return 2;
2130}
2131
2132static void intel_print_wm_latency(struct drm_device *dev,
2133 const char *name,
2134 const uint16_t wm[5])
2135{
2136 int level, max_level = ilk_wm_max_level(dev);
26ec971e
VS		 2137
2138 for (level = 0; level <= max_level; level++) {
2139 unsigned int latency = wm[level];
2140
2141 if (latency == 0) {
2142 DRM_ERROR("%s WM%d latency not provided\n",
2143 name, level);
2144 continue;
2145 }
2146
2147 /* WM1+ latency values in 0.5us units */
2148 if (level > 0)
2149 latency *= 5;
2150
2151 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2152 name, level, wm[level],
2153 latency / 10, latency % 10);
2154 }
2155}
2156
e95a2f75
VS		 2157static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2158 uint16_t wm[5], uint16_t min)
2159{
2160 int level, max_level = ilk_wm_max_level(dev_priv->dev);
2161
2162 if (wm[0] >= min)
2163 return false;
2164
2165 wm[0] = max(wm[0], min);
2166 for (level = 1; level <= max_level; level++)
2167 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
2168
2169 return true;
2170}
2171
2172static void snb_wm_latency_quirk(struct drm_device *dev)
2173{
2174 struct drm_i915_private *dev_priv = dev->dev_private;
2175 bool changed;
2176
2177 /*
2178 * The BIOS provided WM memory latency values are often
2179 * inadequate for high resolution displays. Adjust them.
2180 */
2181 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
2182 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
2183 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
2184
2185 if (!changed)
2186 return;
2187
2188 DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
2189 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
2190 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2191 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
2192}
2193
fa50ad61 2194static void ilk_setup_wm_latency(struct drm_device *dev)
53615a5e
VS		 2195{
2196 struct drm_i915_private *dev_priv = dev->dev_private;
2197
2198 intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
2199
2200 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
2201 sizeof(dev_priv->wm.pri_latency));
2202 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
2203 sizeof(dev_priv->wm.pri_latency));
2204
2205 intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
2206 intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
26ec971e
VS		 2207
2208 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
2209 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2210 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
e95a2f75
VS		 2211
2212 if (IS_GEN6(dev))
2213 snb_wm_latency_quirk(dev);
53615a5e
VS		 2214}
2215
820c1980 2216static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
2a44b76b 2217 struct ilk_pipe_wm_parameters *p)
1011d8c4 2218{
7c4a395f
VS		 2219 struct drm_device *dev = crtc->dev;
2220 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2221 enum pipe pipe = intel_crtc->pipe;
7c4a395f 2222 struct drm_plane *plane;
1011d8c4 2223
2a44b76b
VS		 2224 if (!intel_crtc_active(crtc))
2225 return;
801bcfff 2226
2a44b76b
VS		 2227 p->active = true;
2228 p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal;
2229 p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
2230 p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8;
2231 p->cur.bytes_per_pixel = 4;
2232 p->pri.horiz_pixels = intel_crtc->config.pipe_src_w;
2233 p->cur.horiz_pixels = intel_crtc->cursor_width;
2234 /* TODO: for now, assume primary and cursor planes are always enabled. */
2235 p->pri.enabled = true;
2236 p->cur.enabled = true;
7c4a395f 2237
af2b653b 2238 drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
801bcfff 2239 struct intel_plane *intel_plane = to_intel_plane(plane);
801bcfff 2240
2a44b76b 2241 if (intel_plane->pipe == pipe) {
7c4a395f 2242 p->spr = intel_plane->wm;
2a44b76b
VS		 2243 break;
2244 }
2245 }
2246}
2247
2248static void ilk_compute_wm_config(struct drm_device *dev,
2249 struct intel_wm_config *config)
2250{
2251 struct intel_crtc *intel_crtc;
2252
2253 /* Compute the currently _active_ config */
d3fcc808 2254 for_each_intel_crtc(dev, intel_crtc) {
2a44b76b 2255 const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
cca32e9a 2256
2a44b76b
VS		 2257 if (!wm->pipe_enabled)
2258 continue;
cca32e9a 2259
2a44b76b
VS		 2260 config->sprites_enabled |= wm->sprites_enabled;
2261 config->sprites_scaled |= wm->sprites_scaled;
2262 config->num_pipes_active++;
cca32e9a 2263 }
801bcfff
PZ		 2264}
2265
0b2ae6d7
VS		 2266/* Compute new watermarks for the pipe */
2267static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
820c1980 2268 const struct ilk_pipe_wm_parameters *params,
0b2ae6d7
VS		 2269 struct intel_pipe_wm *pipe_wm)
2270{
2271 struct drm_device *dev = crtc->dev;
d34ff9c6 2272 const struct drm_i915_private *dev_priv = dev->dev_private;
0b2ae6d7
VS		 2273 int level, max_level = ilk_wm_max_level(dev);
2274 /* LP0 watermark maximums depend on this pipe alone */
2275 struct intel_wm_config config = {
2276 .num_pipes_active = 1,
2277 .sprites_enabled = params->spr.enabled,
2278 .sprites_scaled = params->spr.scaled,
2279 };
820c1980 2280 struct ilk_wm_maximums max;
0b2ae6d7 2281
2a44b76b
VS		 2282 pipe_wm->pipe_enabled = params->active;
2283 pipe_wm->sprites_enabled = params->spr.enabled;
2284 pipe_wm->sprites_scaled = params->spr.scaled;
2285
7b39a0b7
VS		 2286 /* ILK/SNB: LP2+ watermarks only w/o sprites */
2287 if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
2288 max_level = 1;
2289
2290 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2291 if (params->spr.scaled)
2292 max_level = 0;
2293
a3cb4048 2294 ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]);
0b2ae6d7 2295
a42a5719 2296 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
ce0e0713 2297 pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
0b2ae6d7 2298
a3cb4048
VS		 2299 /* LP0 watermarks always use 1/2 DDB partitioning */
2300 ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
2301
0b2ae6d7 2302 /* At least LP0 must be valid */
a3cb4048
VS		 2303 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
2304 return false;
2305
2306 ilk_compute_wm_reg_maximums(dev, 1, &max);
2307
2308 for (level = 1; level <= max_level; level++) {
2309 struct intel_wm_level wm = {};
2310
2311 ilk_compute_wm_level(dev_priv, level, params, &wm);
2312
2313 /*
2314 * Disable any watermark level that exceeds the
2315 * register maximums since such watermarks are
2316 * always invalid.
2317 */
2318 if (!ilk_validate_wm_level(level, &max, &wm))
2319 break;
2320
2321 pipe_wm->wm[level] = wm;
2322 }
2323
2324 return true;
0b2ae6d7
VS		 2325}
2326
2327/*
2328 * Merge the watermarks from all active pipes for a specific level.
2329 */
2330static void ilk_merge_wm_level(struct drm_device *dev,
2331 int level,
2332 struct intel_wm_level *ret_wm)
2333{
2334 const struct intel_crtc *intel_crtc;
2335
d52fea5b
VS		 2336 ret_wm->enable = true;
2337
d3fcc808 2338 for_each_intel_crtc(dev, intel_crtc) {
fe392efd
VS		 2339 const struct intel_pipe_wm *active = &intel_crtc->wm.active;
2340 const struct intel_wm_level *wm = &active->wm[level];
2341
2342 if (!active->pipe_enabled)
2343 continue;
0b2ae6d7 2344
d52fea5b
VS		 2345 /*
2346 * The watermark values may have been used in the past,
2347 * so we must maintain them in the registers for some
2348 * time even if the level is now disabled.
2349 */
0b2ae6d7 2350 if (!wm->enable)
d52fea5b 2351 ret_wm->enable = false;
0b2ae6d7
VS		 2352
2353 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2354 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2355 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2356 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2357 }
0b2ae6d7
VS		 2358}
2359
2360/*
2361 * Merge all low power watermarks for all active pipes.
2362 */
2363static void ilk_wm_merge(struct drm_device *dev,
0ba22e26 2364 const struct intel_wm_config *config,
820c1980 2365 const struct ilk_wm_maximums *max,
0b2ae6d7
VS		 2366 struct intel_pipe_wm *merged)
2367{
2368 int level, max_level = ilk_wm_max_level(dev);
d52fea5b 2369 int last_enabled_level = max_level;
0b2ae6d7 2370
0ba22e26
VS		 2371 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
2372 if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
2373 config->num_pipes_active > 1)
2374 return;
2375
6c8b6c28
VS		 2376 /* ILK: FBC WM must be disabled always */
2377 merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
0b2ae6d7
VS		 2378
2379 /* merge each WM1+ level */
2380 for (level = 1; level <= max_level; level++) {
2381 struct intel_wm_level *wm = &merged->wm[level];
2382
2383 ilk_merge_wm_level(dev, level, wm);
2384
d52fea5b
VS		 2385 if (level > last_enabled_level)
2386 wm->enable = false;
2387 else if (!ilk_validate_wm_level(level, max, wm))
2388 /* make sure all following levels get disabled */
2389 last_enabled_level = level - 1;
0b2ae6d7
VS		 2390
2391 /*
2392 * The spec says it is preferred to disable
2393 * FBC WMs instead of disabling a WM level.
2394 */
2395 if (wm->fbc_val > max->fbc) {
d52fea5b
VS		 2396 if (wm->enable)
2397 merged->fbc_wm_enabled = false;
0b2ae6d7
VS		 2398 wm->fbc_val = 0;
2399 }
2400 }
6c8b6c28
VS		 2401
2402 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
2403 /*
2404 * FIXME this is racy. FBC might get enabled later.
2405 * What we should check here is whether FBC can be
2406 * enabled sometime later.
2407 */
2408 if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) {
2409 for (level = 2; level <= max_level; level++) {
2410 struct intel_wm_level *wm = &merged->wm[level];
2411
2412 wm->enable = false;
2413 }
2414 }
0b2ae6d7
VS		 2415}
2416
b380ca3c
VS		 2417static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
2418{
2419 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
2420 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
2421}
2422
a68d68ee
VS		 2423/* The value we need to program into the WM_LPx latency field */
2424static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
2425{
2426 struct drm_i915_private *dev_priv = dev->dev_private;
2427
a42a5719 2428 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
a68d68ee
VS		 2429 return 2 * level;
2430 else
2431 return dev_priv->wm.pri_latency[level];
2432}
2433
820c1980 2434static void ilk_compute_wm_results(struct drm_device *dev,
0362c781 2435 const struct intel_pipe_wm *merged,
609cedef 2436 enum intel_ddb_partitioning partitioning,
820c1980 2437 struct ilk_wm_values *results)
801bcfff 2438{
0b2ae6d7
VS		 2439 struct intel_crtc *intel_crtc;
2440 int level, wm_lp;
cca32e9a 2441
0362c781 2442 results->enable_fbc_wm = merged->fbc_wm_enabled;
609cedef 2443 results->partitioning = partitioning;
cca32e9a 2444
0b2ae6d7 2445 /* LP1+ register values */
cca32e9a 2446 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
1fd527cc 2447 const struct intel_wm_level *r;
801bcfff 2448
b380ca3c 2449 level = ilk_wm_lp_to_level(wm_lp, merged);
0b2ae6d7 2450
0362c781 2451 r = &merged->wm[level];
cca32e9a 2452
d52fea5b
VS		 2453 /*
2454 * Maintain the watermark values even if the level is
2455 * disabled. Doing otherwise could cause underruns.
2456 */
2457 results->wm_lp[wm_lp - 1] =
a68d68ee 2458 (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
416f4727
VS		 2459 (r->pri_val << WM1_LP_SR_SHIFT) |
2460 r->cur_val;
2461
d52fea5b
VS		 2462 if (r->enable)
2463 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
2464
416f4727
VS		 2465 if (INTEL_INFO(dev)->gen >= 8)
2466 results->wm_lp[wm_lp - 1] |=
2467 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
2468 else
2469 results->wm_lp[wm_lp - 1] |=
2470 r->fbc_val << WM1_LP_FBC_SHIFT;
2471
d52fea5b
VS		 2472 /*
2473 * Always set WM1S_LP_EN when spr_val != 0, even if the
2474 * level is disabled. Doing otherwise could cause underruns.
2475 */
6cef2b8a
VS		 2476 if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
2477 WARN_ON(wm_lp != 1);
2478 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
2479 } else
2480 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
cca32e9a 2481 }
801bcfff 2482
0b2ae6d7 2483 /* LP0 register values */
d3fcc808 2484 for_each_intel_crtc(dev, intel_crtc) {
0b2ae6d7
VS		 2485 enum pipe pipe = intel_crtc->pipe;
2486 const struct intel_wm_level *r =
2487 &intel_crtc->wm.active.wm[0];
2488
2489 if (WARN_ON(!r->enable))
2490 continue;
2491
2492 results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
1011d8c4 2493
0b2ae6d7
VS		 2494 results->wm_pipe[pipe] =
2495 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
2496 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
2497 r->cur_val;
801bcfff
PZ		 2498 }
2499}
2500
861f3389
PZ		 2501/* Find the result with the highest level enabled. Check for enable_fbc_wm in
2502 * case both are at the same level. Prefer r1 in case they're the same. */
820c1980 2503static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
198a1e9b
VS		 2504 struct intel_pipe_wm *r1,
2505 struct intel_pipe_wm *r2)
861f3389 2506{
198a1e9b
VS		 2507 int level, max_level = ilk_wm_max_level(dev);
2508 int level1 = 0, level2 = 0;
861f3389 2509
198a1e9b
VS		 2510 for (level = 1; level <= max_level; level++) {
2511 if (r1->wm[level].enable)
2512 level1 = level;
2513 if (r2->wm[level].enable)
2514 level2 = level;
861f3389
PZ		 2515 }
2516
198a1e9b
VS		 2517 if (level1 == level2) {
2518 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
861f3389
PZ		 2519 return r2;
2520 else
2521 return r1;
198a1e9b 2522 } else if (level1 > level2) {
861f3389
PZ		 2523 return r1;
2524 } else {
2525 return r2;
2526 }
2527}
2528
49a687c4
VS		 2529/* dirty bits used to track which watermarks need changes */
2530#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
2531#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
2532#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
2533#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
2534#define WM_DIRTY_FBC (1 << 24)
2535#define WM_DIRTY_DDB (1 << 25)
2536
2537static unsigned int ilk_compute_wm_dirty(struct drm_device *dev,
820c1980
ID		 2538 const struct ilk_wm_values *old,
2539 const struct ilk_wm_values *new)
49a687c4
VS		 2540{
2541 unsigned int dirty = 0;
2542 enum pipe pipe;
2543 int wm_lp;
2544
2545 for_each_pipe(pipe) {
2546 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
2547 dirty |= WM_DIRTY_LINETIME(pipe);
2548 /* Must disable LP1+ watermarks too */
2549 dirty |= WM_DIRTY_LP_ALL;
2550 }
2551
2552 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
2553 dirty |= WM_DIRTY_PIPE(pipe);
2554 /* Must disable LP1+ watermarks too */
2555 dirty |= WM_DIRTY_LP_ALL;
2556 }
2557 }
2558
2559 if (old->enable_fbc_wm != new->enable_fbc_wm) {
2560 dirty |= WM_DIRTY_FBC;
2561 /* Must disable LP1+ watermarks too */
2562 dirty |= WM_DIRTY_LP_ALL;
2563 }
2564
2565 if (old->partitioning != new->partitioning) {
2566 dirty |= WM_DIRTY_DDB;
2567 /* Must disable LP1+ watermarks too */
2568 dirty |= WM_DIRTY_LP_ALL;
2569 }
2570
2571 /* LP1+ watermarks already deemed dirty, no need to continue */
2572 if (dirty & WM_DIRTY_LP_ALL)
2573 return dirty;
2574
2575 /* Find the lowest numbered LP1+ watermark in need of an update... */
2576 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2577 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
2578 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
2579 break;
2580 }
2581
2582 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
2583 for (; wm_lp <= 3; wm_lp++)
2584 dirty |= WM_DIRTY_LP(wm_lp);
2585
2586 return dirty;
2587}
2588
8553c18e
VS		 2589static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
2590 unsigned int dirty)
801bcfff 2591{
820c1980 2592 struct ilk_wm_values *previous = &dev_priv->wm.hw;
8553c18e 2593 bool changed = false;
801bcfff 2594
facd619b
VS		 2595 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
2596 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
2597 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
8553c18e 2598 changed = true;
facd619b
VS		 2599 }
2600 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
2601 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
2602 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
8553c18e 2603 changed = true;
facd619b
VS		 2604 }
2605 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
2606 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
2607 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
8553c18e 2608 changed = true;
facd619b 2609 }
801bcfff 2610
facd619b
VS		 2611 /*
2612 * Don't touch WM1S_LP_EN here.
2613 * Doing so could cause underruns.
2614 */
6cef2b8a 2615
8553c18e
VS		 2616 return changed;
2617}
2618
2619/*
2620 * The spec says we shouldn't write when we don't need, because every write
2621 * causes WMs to be re-evaluated, expending some power.
2622 */
820c1980
ID		 2623static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2624 struct ilk_wm_values *results)
8553c18e
VS		 2625{
2626 struct drm_device *dev = dev_priv->dev;
820c1980 2627 struct ilk_wm_values *previous = &dev_priv->wm.hw;
8553c18e
VS		 2628 unsigned int dirty;
2629 uint32_t val;
2630
2631 dirty = ilk_compute_wm_dirty(dev, previous, results);
2632 if (!dirty)
2633 return;
2634
2635 _ilk_disable_lp_wm(dev_priv, dirty);
2636
49a687c4 2637 if (dirty & WM_DIRTY_PIPE(PIPE_A))
801bcfff 2638 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
49a687c4 2639 if (dirty & WM_DIRTY_PIPE(PIPE_B))
801bcfff 2640 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
49a687c4 2641 if (dirty & WM_DIRTY_PIPE(PIPE_C))
801bcfff
PZ		 2642 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
2643
49a687c4 2644 if (dirty & WM_DIRTY_LINETIME(PIPE_A))
801bcfff 2645 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
49a687c4 2646 if (dirty & WM_DIRTY_LINETIME(PIPE_B))
801bcfff 2647 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
49a687c4 2648 if (dirty & WM_DIRTY_LINETIME(PIPE_C))
801bcfff
PZ		 2649 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
2650
49a687c4 2651 if (dirty & WM_DIRTY_DDB) {
a42a5719 2652 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
ac9545fd
VS		 2653 val = I915_READ(WM_MISC);
2654 if (results->partitioning == INTEL_DDB_PART_1_2)
2655 val &= ~WM_MISC_DATA_PARTITION_5_6;
2656 else
2657 val |= WM_MISC_DATA_PARTITION_5_6;
2658 I915_WRITE(WM_MISC, val);
2659 } else {
2660 val = I915_READ(DISP_ARB_CTL2);
2661 if (results->partitioning == INTEL_DDB_PART_1_2)
2662 val &= ~DISP_DATA_PARTITION_5_6;
2663 else
2664 val |= DISP_DATA_PARTITION_5_6;
2665 I915_WRITE(DISP_ARB_CTL2, val);
2666 }
1011d8c4
PZ		 2667 }
2668
49a687c4 2669 if (dirty & WM_DIRTY_FBC) {
cca32e9a
PZ		 2670 val = I915_READ(DISP_ARB_CTL);
2671 if (results->enable_fbc_wm)
2672 val &= ~DISP_FBC_WM_DIS;
2673 else
2674 val |= DISP_FBC_WM_DIS;
2675 I915_WRITE(DISP_ARB_CTL, val);
2676 }
2677
954911eb
ID		 2678 if (dirty & WM_DIRTY_LP(1) &&
2679 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
2680 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
2681
2682 if (INTEL_INFO(dev)->gen >= 7) {
6cef2b8a
VS		 2683 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
2684 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
2685 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
2686 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
2687 }
801bcfff 2688
facd619b 2689 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
801bcfff 2690 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
facd619b 2691 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
801bcfff 2692 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
facd619b 2693 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
801bcfff 2694 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
609cedef
VS		 2695
2696 dev_priv->wm.hw = *results;
801bcfff
PZ		 2697}
2698
8553c18e
VS		 2699static bool ilk_disable_lp_wm(struct drm_device *dev)
2700{
2701 struct drm_i915_private *dev_priv = dev->dev_private;
2702
2703 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
2704}
2705
820c1980 2706static void ilk_update_wm(struct drm_crtc *crtc)
801bcfff 2707{
7c4a395f 2708 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
46ba614c 2709 struct drm_device *dev = crtc->dev;
801bcfff 2710 struct drm_i915_private *dev_priv = dev->dev_private;
820c1980
ID		 2711 struct ilk_wm_maximums max;
2712 struct ilk_pipe_wm_parameters params = {};
2713 struct ilk_wm_values results = {};
77c122bc 2714 enum intel_ddb_partitioning partitioning;
7c4a395f 2715 struct intel_pipe_wm pipe_wm = {};
198a1e9b 2716 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
a485bfb8 2717 struct intel_wm_config config = {};
7c4a395f 2718
2a44b76b 2719 ilk_compute_wm_parameters(crtc, &params);
7c4a395f
VS		 2720
2721 intel_compute_pipe_wm(crtc, &params, &pipe_wm);
2722
2723 if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
2724 return;
861f3389 2725
7c4a395f 2726 intel_crtc->wm.active = pipe_wm;
861f3389 2727
2a44b76b
VS		 2728 ilk_compute_wm_config(dev, &config);
2729
34982fe1 2730 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
0ba22e26 2731 ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
a485bfb8
VS		 2732
2733 /* 5/6 split only in single pipe config on IVB+ */
ec98c8d1
VS		 2734 if (INTEL_INFO(dev)->gen >= 7 &&
2735 config.num_pipes_active == 1 && config.sprites_enabled) {
34982fe1 2736 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
0ba22e26 2737 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
0362c781 2738
820c1980 2739 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
861f3389 2740 } else {
198a1e9b 2741 best_lp_wm = &lp_wm_1_2;
861f3389
PZ		 2742 }
2743
198a1e9b 2744 partitioning = (best_lp_wm == &lp_wm_1_2) ?
77c122bc 2745 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
801bcfff 2746
820c1980 2747 ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
609cedef 2748
820c1980 2749 ilk_write_wm_values(dev_priv, &results);
1011d8c4
PZ		 2750}
2751
820c1980 2752static void ilk_update_sprite_wm(struct drm_plane *plane,
adf3d35e 2753 struct drm_crtc *crtc,
526682e9 2754 uint32_t sprite_width, int pixel_size,
bdd57d03 2755 bool enabled, bool scaled)
526682e9 2756{
8553c18e 2757 struct drm_device *dev = plane->dev;
adf3d35e 2758 struct intel_plane *intel_plane = to_intel_plane(plane);
526682e9 2759
adf3d35e
VS		 2760 intel_plane->wm.enabled = enabled;
2761 intel_plane->wm.scaled = scaled;
2762 intel_plane->wm.horiz_pixels = sprite_width;
2763 intel_plane->wm.bytes_per_pixel = pixel_size;
526682e9 2764
8553c18e
VS		 2765 /*
2766 * IVB workaround: must disable low power watermarks for at least
2767 * one frame before enabling scaling. LP watermarks can be re-enabled
2768 * when scaling is disabled.
2769 *
2770 * WaCxSRDisabledForSpriteScaling:ivb
2771 */
2772 if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
2773 intel_wait_for_vblank(dev, intel_plane->pipe);
2774
820c1980 2775 ilk_update_wm(crtc);
526682e9
PZ		 2776}
2777
243e6a44
VS		 2778static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
2779{
2780 struct drm_device *dev = crtc->dev;
2781 struct drm_i915_private *dev_priv = dev->dev_private;
820c1980 2782 struct ilk_wm_values *hw = &dev_priv->wm.hw;
243e6a44
VS		 2783 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2784 struct intel_pipe_wm *active = &intel_crtc->wm.active;
2785 enum pipe pipe = intel_crtc->pipe;
2786 static const unsigned int wm0_pipe_reg[] = {
2787 [PIPE_A] = WM0_PIPEA_ILK,
2788 [PIPE_B] = WM0_PIPEB_ILK,
2789 [PIPE_C] = WM0_PIPEC_IVB,
2790 };
2791
2792 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
a42a5719 2793 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
ce0e0713 2794 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
243e6a44 2795
2a44b76b
VS		 2796 active->pipe_enabled = intel_crtc_active(crtc);
2797
2798 if (active->pipe_enabled) {
243e6a44
VS		 2799 u32 tmp = hw->wm_pipe[pipe];
2800
2801 /*
2802 * For active pipes LP0 watermark is marked as
2803 * enabled, and LP1+ watermaks as disabled since
2804 * we can't really reverse compute them in case
2805 * multiple pipes are active.
2806 */
2807 active->wm[0].enable = true;
2808 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
2809 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
2810 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
2811 active->linetime = hw->wm_linetime[pipe];
2812 } else {
2813 int level, max_level = ilk_wm_max_level(dev);
2814
2815 /*
2816 * For inactive pipes, all watermark levels
2817 * should be marked as enabled but zeroed,
2818 * which is what we'd compute them to.
2819 */
2820 for (level = 0; level <= max_level; level++)
2821 active->wm[level].enable = true;
2822 }
2823}
2824
2825void ilk_wm_get_hw_state(struct drm_device *dev)
2826{
2827 struct drm_i915_private *dev_priv = dev->dev_private;
820c1980 2828 struct ilk_wm_values *hw = &dev_priv->wm.hw;
243e6a44
VS		 2829 struct drm_crtc *crtc;
2830
70e1e0ec 2831 for_each_crtc(dev, crtc)
243e6a44
VS		 2832 ilk_pipe_wm_get_hw_state(crtc);
2833
2834 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
2835 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
2836 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
2837
2838 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
cfa7698b
VS		 2839 if (INTEL_INFO(dev)->gen >= 7) {
2840 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
2841 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
2842 }
243e6a44 2843
a42a5719 2844 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
ac9545fd
VS		 2845 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
2846 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
2847 else if (IS_IVYBRIDGE(dev))
2848 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
2849 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
243e6a44
VS		 2850
2851 hw->enable_fbc_wm =
2852 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
2853}
2854
b445e3b0
ED		 2855/**
2856 * intel_update_watermarks - update FIFO watermark values based on current modes
2857 *
2858 * Calculate watermark values for the various WM regs based on current mode
2859 * and plane configuration.
2860 *
2861 * There are several cases to deal with here:
2862 * - normal (i.e. non-self-refresh)
2863 * - self-refresh (SR) mode
2864 * - lines are large relative to FIFO size (buffer can hold up to 2)
2865 * - lines are small relative to FIFO size (buffer can hold more than 2
2866 * lines), so need to account for TLB latency
2867 *
2868 * The normal calculation is:
2869 * watermark = dotclock * bytes per pixel * latency
2870 * where latency is platform & configuration dependent (we assume pessimal
2871 * values here).
2872 *
2873 * The SR calculation is:
2874 * watermark = (trunc(latency/line time)+1) * surface width *
2875 * bytes per pixel
2876 * where
2877 * line time = htotal / dotclock
2878 * surface width = hdisplay for normal plane and 64 for cursor
2879 * and latency is assumed to be high, as above.
2880 *
2881 * The final value programmed to the register should always be rounded up,
2882 * and include an extra 2 entries to account for clock crossings.
2883 *
2884 * We don't use the sprite, so we can ignore that. And on Crestline we have
2885 * to set the non-SR watermarks to 8.
2886 */
46ba614c 2887void intel_update_watermarks(struct drm_crtc *crtc)
b445e3b0 2888{
46ba614c 2889 struct drm_i915_private *dev_priv = crtc->dev->dev_private;
b445e3b0
ED		 2890
2891 if (dev_priv->display.update_wm)
46ba614c 2892 dev_priv->display.update_wm(crtc);
b445e3b0
ED		 2893}
2894
adf3d35e
VS		 2895void intel_update_sprite_watermarks(struct drm_plane *plane,
2896 struct drm_crtc *crtc,
4c4ff43a 2897 uint32_t sprite_width, int pixel_size,
39db4a4d 2898 bool enabled, bool scaled)
b445e3b0 2899{
adf3d35e 2900 struct drm_i915_private *dev_priv = plane->dev->dev_private;
b445e3b0
ED		 2901
2902 if (dev_priv->display.update_sprite_wm)
adf3d35e 2903 dev_priv->display.update_sprite_wm(plane, crtc, sprite_width,
39db4a4d 2904 pixel_size, enabled, scaled);
b445e3b0
ED		 2905}
2906
2b4e57bd
ED		 2907static struct drm_i915_gem_object *
2908intel_alloc_context_page(struct drm_device *dev)
2909{
2910 struct drm_i915_gem_object *ctx;
2911 int ret;
2912
2913 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2914
2915 ctx = i915_gem_alloc_object(dev, 4096);
2916 if (!ctx) {
2917 DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
2918 return NULL;
2919 }
2920
c69766f2 2921 ret = i915_gem_obj_ggtt_pin(ctx, 4096, 0);
2b4e57bd
ED		 2922 if (ret) {
2923 DRM_ERROR("failed to pin power context: %d\n", ret);
2924 goto err_unref;
2925 }
2926
2927 ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
2928 if (ret) {
2929 DRM_ERROR("failed to set-domain on power context: %d\n", ret);
2930 goto err_unpin;
2931 }
2932
2933 return ctx;
2934
2935err_unpin:
d7f46fc4 2936 i915_gem_object_ggtt_unpin(ctx);
2b4e57bd
ED		 2937err_unref:
2938 drm_gem_object_unreference(&ctx->base);
2b4e57bd
ED		 2939 return NULL;
2940}
2941
9270388e
DV		 2942/**
2943 * Lock protecting IPS related data structures
9270388e
DV		 2944 */
2945DEFINE_SPINLOCK(mchdev_lock);
2946
2947/* Global for IPS driver to get at the current i915 device. Protected by
2948 * mchdev_lock. */
2949static struct drm_i915_private *i915_mch_dev;
2950
2b4e57bd
ED		 2951bool ironlake_set_drps(struct drm_device *dev, u8 val)
2952{
2953 struct drm_i915_private *dev_priv = dev->dev_private;
2954 u16 rgvswctl;
2955
9270388e
DV		 2956 assert_spin_locked(&mchdev_lock);
2957
2b4e57bd
ED		 2958 rgvswctl = I915_READ16(MEMSWCTL);
2959 if (rgvswctl & MEMCTL_CMD_STS) {
2960 DRM_DEBUG("gpu busy, RCS change rejected\n");
2961 return false; /* still busy with another command */
2962 }
2963
2964 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
2965 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
2966 I915_WRITE16(MEMSWCTL, rgvswctl);
2967 POSTING_READ16(MEMSWCTL);
2968
2969 rgvswctl |= MEMCTL_CMD_STS;
2970 I915_WRITE16(MEMSWCTL, rgvswctl);
2971
2972 return true;
2973}
2974
8090c6b9 2975static void ironlake_enable_drps(struct drm_device *dev)
2b4e57bd
ED		 2976{
2977 struct drm_i915_private *dev_priv = dev->dev_private;
2978 u32 rgvmodectl = I915_READ(MEMMODECTL);
2979 u8 fmax, fmin, fstart, vstart;
2980
9270388e
DV		 2981 spin_lock_irq(&mchdev_lock);
2982
2b4e57bd
ED		 2983 /* Enable temp reporting */
2984 I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
2985 I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
2986
2987 /* 100ms RC evaluation intervals */
2988 I915_WRITE(RCUPEI, 100000);
2989 I915_WRITE(RCDNEI, 100000);
2990
2991 /* Set max/min thresholds to 90ms and 80ms respectively */
2992 I915_WRITE(RCBMAXAVG, 90000);
2993 I915_WRITE(RCBMINAVG, 80000);
2994
2995 I915_WRITE(MEMIHYST, 1);
2996
2997 /* Set up min, max, and cur for interrupt handling */
2998 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
2999 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
3000 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
3001 MEMMODE_FSTART_SHIFT;
3002
3003 vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
3004 PXVFREQ_PX_SHIFT;
3005
20e4d407
DV		 3006 dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
3007 dev_priv->ips.fstart = fstart;
2b4e57bd 3008
20e4d407
DV		 3009 dev_priv->ips.max_delay = fstart;
3010 dev_priv->ips.min_delay = fmin;
3011 dev_priv->ips.cur_delay = fstart;
2b4e57bd
ED		 3012
3013 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
3014 fmax, fmin, fstart);
3015
3016 I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
3017
3018 /*
3019 * Interrupts will be enabled in ironlake_irq_postinstall
3020 */
3021
3022 I915_WRITE(VIDSTART, vstart);
3023 POSTING_READ(VIDSTART);
3024
3025 rgvmodectl |= MEMMODE_SWMODE_EN;
3026 I915_WRITE(MEMMODECTL, rgvmodectl);
3027
9270388e 3028 if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
2b4e57bd 3029 DRM_ERROR("stuck trying to change perf mode\n");
9270388e 3030 mdelay(1);
2b4e57bd
ED		 3031
3032 ironlake_set_drps(dev, fstart);
3033
20e4d407 3034 dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
2b4e57bd 3035 I915_READ(0x112e0);
20e4d407
DV		 3036 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
3037 dev_priv->ips.last_count2 = I915_READ(0x112f4);
3038 getrawmonotonic(&dev_priv->ips.last_time2);
9270388e
DV		 3039
3040 spin_unlock_irq(&mchdev_lock);
2b4e57bd
ED		 3041}
3042
8090c6b9 3043static void ironlake_disable_drps(struct drm_device *dev)
2b4e57bd
ED		 3044{
3045 struct drm_i915_private *dev_priv = dev->dev_private;
9270388e
DV		 3046 u16 rgvswctl;
3047
3048 spin_lock_irq(&mchdev_lock);
3049
3050 rgvswctl = I915_READ16(MEMSWCTL);
2b4e57bd
ED		 3051
3052 /* Ack interrupts, disable EFC interrupt */
3053 I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
3054 I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
3055 I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
3056 I915_WRITE(DEIIR, DE_PCU_EVENT);
3057 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
3058
3059 /* Go back to the starting frequency */
20e4d407 3060 ironlake_set_drps(dev, dev_priv->ips.fstart);
9270388e 3061 mdelay(1);
2b4e57bd
ED		 3062 rgvswctl |= MEMCTL_CMD_STS;
3063 I915_WRITE(MEMSWCTL, rgvswctl);
9270388e 3064 mdelay(1);
2b4e57bd 3065
9270388e 3066 spin_unlock_irq(&mchdev_lock);
2b4e57bd
ED		 3067}
3068
acbe9475
DV		 3069/* There's a funny hw issue where the hw returns all 0 when reading from
3070 * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
3071 * ourselves, instead of doing a rmw cycle (which might result in us clearing
3072 * all limits and the gpu stuck at whatever frequency it is at atm).
3073 */
6917c7b9 3074static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val)
2b4e57bd 3075{
7b9e0ae6 3076 u32 limits;
2b4e57bd 3077
20b46e59
DV		 3078 /* Only set the down limit when we've reached the lowest level to avoid
3079 * getting more interrupts, otherwise leave this clear. This prevents a
3080 * race in the hw when coming out of rc6: There's a tiny window where
3081 * the hw runs at the minimal clock before selecting the desired
3082 * frequency, if the down threshold expires in that window we will not
3083 * receive a down interrupt. */
b39fb297
BW		 3084 limits = dev_priv->rps.max_freq_softlimit << 24;
3085 if (val <= dev_priv->rps.min_freq_softlimit)
3086 limits |= dev_priv->rps.min_freq_softlimit << 16;
20b46e59
DV		 3087
3088 return limits;
3089}
3090
dd75fdc8
CW		 3091static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
3092{
3093 int new_power;
3094
3095 new_power = dev_priv->rps.power;
3096 switch (dev_priv->rps.power) {
3097 case LOW_POWER:
b39fb297 3098 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
dd75fdc8
CW		 3099 new_power = BETWEEN;
3100 break;
3101
3102 case BETWEEN:
b39fb297 3103 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
dd75fdc8 3104 new_power = LOW_POWER;
b39fb297 3105 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
dd75fdc8
CW		 3106 new_power = HIGH_POWER;
3107 break;
3108
3109 case HIGH_POWER:
b39fb297 3110 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
dd75fdc8
CW		 3111 new_power = BETWEEN;
3112 break;
3113 }
3114 /* Max/min bins are special */
b39fb297 3115 if (val == dev_priv->rps.min_freq_softlimit)
dd75fdc8 3116 new_power = LOW_POWER;
b39fb297 3117 if (val == dev_priv->rps.max_freq_softlimit)
dd75fdc8
CW		 3118 new_power = HIGH_POWER;
3119 if (new_power == dev_priv->rps.power)
3120 return;
3121
3122 /* Note the units here are not exactly 1us, but 1280ns. */
3123 switch (new_power) {
3124 case LOW_POWER:
3125 /* Upclock if more than 95% busy over 16ms */
3126 I915_WRITE(GEN6_RP_UP_EI, 12500);
3127 I915_WRITE(GEN6_RP_UP_THRESHOLD, 11800);
3128
3129 /* Downclock if less than 85% busy over 32ms */
3130 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3131 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 21250);
3132
3133 I915_WRITE(GEN6_RP_CONTROL,
3134 GEN6_RP_MEDIA_TURBO |
3135 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3136 GEN6_RP_MEDIA_IS_GFX |
3137 GEN6_RP_ENABLE |
3138 GEN6_RP_UP_BUSY_AVG |
3139 GEN6_RP_DOWN_IDLE_AVG);
3140 break;
3141
3142 case BETWEEN:
3143 /* Upclock if more than 90% busy over 13ms */
3144 I915_WRITE(GEN6_RP_UP_EI, 10250);
3145 I915_WRITE(GEN6_RP_UP_THRESHOLD, 9225);
3146
3147 /* Downclock if less than 75% busy over 32ms */
3148 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3149 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 18750);
3150
3151 I915_WRITE(GEN6_RP_CONTROL,
3152 GEN6_RP_MEDIA_TURBO |
3153 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3154 GEN6_RP_MEDIA_IS_GFX |
3155 GEN6_RP_ENABLE |
3156 GEN6_RP_UP_BUSY_AVG |
3157 GEN6_RP_DOWN_IDLE_AVG);
3158 break;
3159
3160 case HIGH_POWER:
3161 /* Upclock if more than 85% busy over 10ms */
3162 I915_WRITE(GEN6_RP_UP_EI, 8000);
3163 I915_WRITE(GEN6_RP_UP_THRESHOLD, 6800);
3164
3165 /* Downclock if less than 60% busy over 32ms */
3166 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3167 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 15000);
3168
3169 I915_WRITE(GEN6_RP_CONTROL,
3170 GEN6_RP_MEDIA_TURBO |
3171 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3172 GEN6_RP_MEDIA_IS_GFX |
3173 GEN6_RP_ENABLE |
3174 GEN6_RP_UP_BUSY_AVG |
3175 GEN6_RP_DOWN_IDLE_AVG);
3176 break;
3177 }
3178
3179 dev_priv->rps.power = new_power;
3180 dev_priv->rps.last_adj = 0;
3181}
3182
2876ce73
CW		 3183static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
3184{
3185 u32 mask = 0;
3186
3187 if (val > dev_priv->rps.min_freq_softlimit)
3188 mask |= GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
3189 if (val < dev_priv->rps.max_freq_softlimit)
3190 mask |= GEN6_PM_RP_UP_THRESHOLD;
3191
3192 /* IVB and SNB hard hangs on looping batchbuffer
3193 * if GEN6_PM_UP_EI_EXPIRED is masked.
3194 */
3195 if (INTEL_INFO(dev_priv->dev)->gen <= 7 && !IS_HASWELL(dev_priv->dev))
3196 mask |= GEN6_PM_RP_UP_EI_EXPIRED;
3197
baccd458
D		 3198 if (IS_GEN8(dev_priv->dev))
3199 mask |= GEN8_PMINTR_REDIRECT_TO_NON_DISP;
3200
2876ce73
CW		 3201 return ~mask;
3202}
3203
b8a5ff8d
JM		 3204/* gen6_set_rps is called to update the frequency request, but should also be
3205 * called when the range (min_delay and max_delay) is modified so that we can
3206 * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
20b46e59
DV		 3207void gen6_set_rps(struct drm_device *dev, u8 val)
3208{
3209 struct drm_i915_private *dev_priv = dev->dev_private;
7b9e0ae6 3210
4fc688ce 3211 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
b39fb297
BW		 3212 WARN_ON(val > dev_priv->rps.max_freq_softlimit);
3213 WARN_ON(val < dev_priv->rps.min_freq_softlimit);
004777cb 3214
eb64cad1
CW		 3215 /* min/max delay may still have been modified so be sure to
3216 * write the limits value.
3217 */
3218 if (val != dev_priv->rps.cur_freq) {
3219 gen6_set_rps_thresholds(dev_priv, val);
b8a5ff8d 3220
50e6a2a7 3221 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
eb64cad1
CW		 3222 I915_WRITE(GEN6_RPNSWREQ,
3223 HSW_FREQUENCY(val));
3224 else
3225 I915_WRITE(GEN6_RPNSWREQ,
3226 GEN6_FREQUENCY(val) |
3227 GEN6_OFFSET(0) |
3228 GEN6_AGGRESSIVE_TURBO);
b8a5ff8d 3229 }
7b9e0ae6 3230
7b9e0ae6
CW		 3231 /* Make sure we continue to get interrupts
3232 * until we hit the minimum or maximum frequencies.
3233 */
eb64cad1 3234 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, gen6_rps_limits(dev_priv, val));
2876ce73 3235 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
7b9e0ae6 3236
d5570a72
BW		 3237 POSTING_READ(GEN6_RPNSWREQ);
3238
b39fb297 3239 dev_priv->rps.cur_freq = val;
be2cde9a 3240 trace_intel_gpu_freq_change(val * 50);
2b4e57bd
ED		 3241}
3242
76c3552f
D		 3243/* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down
3244 *
3245 * * If Gfx is Idle, then
3246 * 1. Mask Turbo interrupts
3247 * 2. Bring up Gfx clock
3248 * 3. Change the freq to Rpn and wait till P-Unit updates freq
3249 * 4. Clear the Force GFX CLK ON bit so that Gfx can down
3250 * 5. Unmask Turbo interrupts
3251*/
3252static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
3253{
5549d25f
D		 3254 struct drm_device *dev = dev_priv->dev;
3255
3256 /* Latest VLV doesn't need to force the gfx clock */
3257 if (dev->pdev->revision >= 0xd) {
3258 valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
3259 return;
3260 }
3261
76c3552f
D		 3262 /*
3263 * When we are idle. Drop to min voltage state.
3264 */
3265
b39fb297 3266 if (dev_priv->rps.cur_freq <= dev_priv->rps.min_freq_softlimit)
76c3552f
D		 3267 return;
3268
3269 /* Mask turbo interrupt so that they will not come in between */
3270 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
3271
650ad970 3272 vlv_force_gfx_clock(dev_priv, true);
76c3552f 3273
b39fb297 3274 dev_priv->rps.cur_freq = dev_priv->rps.min_freq_softlimit;
76c3552f
D		 3275
3276 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ,
b39fb297 3277 dev_priv->rps.min_freq_softlimit);
76c3552f
D		 3278
3279 if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS))
3280 & GENFREQSTATUS) == 0, 5))
3281 DRM_ERROR("timed out waiting for Punit\n");
3282
650ad970 3283 vlv_force_gfx_clock(dev_priv, false);
76c3552f 3284
31685c25
D		 3285 if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
3286 I915_WRITE(GEN6_PMINTRMSK, ~dev_priv->pm_rps_events);
3287 else
3288 I915_WRITE(GEN6_PMINTRMSK,
3289 gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
76c3552f
D		 3290}
3291
b29c19b6
CW		 3292void gen6_rps_idle(struct drm_i915_private *dev_priv)
3293{
691bb717
DL		 3294 struct drm_device *dev = dev_priv->dev;
3295
b29c19b6 3296 mutex_lock(&dev_priv->rps.hw_lock);
c0951f0c 3297 if (dev_priv->rps.enabled) {
691bb717 3298 if (IS_VALLEYVIEW(dev))
76c3552f 3299 vlv_set_rps_idle(dev_priv);
c0951f0c 3300 else
b39fb297 3301 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
c0951f0c
CW		 3302 dev_priv->rps.last_adj = 0;
3303 }
b29c19b6
CW		 3304 mutex_unlock(&dev_priv->rps.hw_lock);
3305}
3306
3307void gen6_rps_boost(struct drm_i915_private *dev_priv)
3308{
691bb717
DL		 3309 struct drm_device *dev = dev_priv->dev;
3310
b29c19b6 3311 mutex_lock(&dev_priv->rps.hw_lock);
c0951f0c 3312 if (dev_priv->rps.enabled) {
691bb717 3313 if (IS_VALLEYVIEW(dev))
b39fb297 3314 valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
c0951f0c 3315 else
b39fb297 3316 gen6_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
c0951f0c
CW		 3317 dev_priv->rps.last_adj = 0;
3318 }
b29c19b6
CW		 3319 mutex_unlock(&dev_priv->rps.hw_lock);
3320}
3321
0a073b84
JB		 3322void valleyview_set_rps(struct drm_device *dev, u8 val)
3323{
3324 struct drm_i915_private *dev_priv = dev->dev_private;
7a67092a 3325
0a073b84 3326 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
b39fb297
BW		 3327 WARN_ON(val > dev_priv->rps.max_freq_softlimit);
3328 WARN_ON(val < dev_priv->rps.min_freq_softlimit);
0a073b84 3329
73008b98 3330 DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
b39fb297
BW		 3331 vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
3332 dev_priv->rps.cur_freq,
2ec3815f 3333 vlv_gpu_freq(dev_priv, val), val);
0a073b84 3334
2876ce73
CW		 3335 if (val != dev_priv->rps.cur_freq)
3336 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
0a073b84 3337
09c87db8 3338 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
0a073b84 3339
b39fb297 3340 dev_priv->rps.cur_freq = val;
2ec3815f 3341 trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val));
0a073b84
JB		 3342}
3343
0961021a
BW		 3344static void gen8_disable_rps_interrupts(struct drm_device *dev)
3345{
3346 struct drm_i915_private *dev_priv = dev->dev_private;
3347
992f191f 3348 I915_WRITE(GEN6_PMINTRMSK, ~GEN8_PMINTR_REDIRECT_TO_NON_DISP);
0961021a
BW		 3349 I915_WRITE(GEN8_GT_IER(2), I915_READ(GEN8_GT_IER(2)) &
3350 ~dev_priv->pm_rps_events);
3351 /* Complete PM interrupt masking here doesn't race with the rps work
3352 * item again unmasking PM interrupts because that is using a different
3353 * register (GEN8_GT_IMR(2)) to mask PM interrupts. The only risk is in
3354 * leaving stale bits in GEN8_GT_IIR(2) and GEN8_GT_IMR(2) which
3355 * gen8_enable_rps will clean up. */
3356
3357 spin_lock_irq(&dev_priv->irq_lock);
3358 dev_priv->rps.pm_iir = 0;
3359 spin_unlock_irq(&dev_priv->irq_lock);
3360
3361 I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events);
3362}
3363
44fc7d5c 3364static void gen6_disable_rps_interrupts(struct drm_device *dev)
2b4e57bd
ED		 3365{
3366 struct drm_i915_private *dev_priv = dev->dev_private;
3367
2b4e57bd 3368 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
a6706b45
D		 3369 I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) &
3370 ~dev_priv->pm_rps_events);
2b4e57bd
ED		 3371 /* Complete PM interrupt masking here doesn't race with the rps work
3372 * item again unmasking PM interrupts because that is using a different
3373 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
3374 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
3375
59cdb63d 3376 spin_lock_irq(&dev_priv->irq_lock);
c6a828d3 3377 dev_priv->rps.pm_iir = 0;
59cdb63d 3378 spin_unlock_irq(&dev_priv->irq_lock);
2b4e57bd 3379
a6706b45 3380 I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events);
2b4e57bd
ED		 3381}
3382
44fc7d5c 3383static void gen6_disable_rps(struct drm_device *dev)
d20d4f0c
JB		 3384{
3385 struct drm_i915_private *dev_priv = dev->dev_private;
3386
3387 I915_WRITE(GEN6_RC_CONTROL, 0);
44fc7d5c 3388 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
d20d4f0c 3389
0961021a
BW		 3390 if (IS_BROADWELL(dev))
3391 gen8_disable_rps_interrupts(dev);
3392 else
3393 gen6_disable_rps_interrupts(dev);
44fc7d5c
DV		 3394}
3395
38807746
D		 3396static void cherryview_disable_rps(struct drm_device *dev)
3397{
3398 struct drm_i915_private *dev_priv = dev->dev_private;
3399
3400 I915_WRITE(GEN6_RC_CONTROL, 0);
3401}
3402
44fc7d5c
DV		 3403static void valleyview_disable_rps(struct drm_device *dev)
3404{
3405 struct drm_i915_private *dev_priv = dev->dev_private;
3406
3407 I915_WRITE(GEN6_RC_CONTROL, 0);
d20d4f0c 3408
44fc7d5c 3409 gen6_disable_rps_interrupts(dev);
d20d4f0c
JB		 3410}
3411
dc39fff7
BW		 3412static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
3413{
91ca689a
ID		 3414 if (IS_VALLEYVIEW(dev)) {
3415 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1)))
3416 mode = GEN6_RC_CTL_RC6_ENABLE;
3417 else
3418 mode = 0;
3419 }
dc39fff7 3420 DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
1c79b42f
BW		 3421 (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
3422 (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
3423 (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
dc39fff7
BW		 3424}
3425
e6069ca8 3426static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
2b4e57bd 3427{
eb4926e4
DL		 3428 /* No RC6 before Ironlake */
3429 if (INTEL_INFO(dev)->gen < 5)
3430 return 0;
3431
e6069ca8
ID		 3432 /* RC6 is only on Ironlake mobile not on desktop */
3433 if (INTEL_INFO(dev)->gen == 5 && !IS_IRONLAKE_M(dev))
3434 return 0;
3435
456470eb 3436 /* Respect the kernel parameter if it is set */
e6069ca8
ID		 3437 if (enable_rc6 >= 0) {
3438 int mask;
3439
3440 if (INTEL_INFO(dev)->gen == 6 || IS_IVYBRIDGE(dev))
3441 mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE |
3442 INTEL_RC6pp_ENABLE;
3443 else
3444 mask = INTEL_RC6_ENABLE;
3445
3446 if ((enable_rc6 & mask) != enable_rc6)
3447 DRM_INFO("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
8fd9c1a9 3448 enable_rc6 & mask, enable_rc6, mask);
e6069ca8
ID		 3449
3450 return enable_rc6 & mask;
3451 }
2b4e57bd 3452
6567d748
CW		 3453 /* Disable RC6 on Ironlake */
3454 if (INTEL_INFO(dev)->gen == 5)
3455 return 0;
2b4e57bd 3456
8bade1ad 3457 if (IS_IVYBRIDGE(dev))
cca84a1f 3458 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
8bade1ad
BW		 3459
3460 return INTEL_RC6_ENABLE;
2b4e57bd
ED		 3461}
3462
e6069ca8
ID		 3463int intel_enable_rc6(const struct drm_device *dev)
3464{
3465 return i915.enable_rc6;
3466}
3467
0961021a
BW		 3468static void gen8_enable_rps_interrupts(struct drm_device *dev)
3469{
3470 struct drm_i915_private *dev_priv = dev->dev_private;
3471
3472 spin_lock_irq(&dev_priv->irq_lock);
3473 WARN_ON(dev_priv->rps.pm_iir);
3474 bdw_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
3475 I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events);
3476 spin_unlock_irq(&dev_priv->irq_lock);
3477}
3478
44fc7d5c
DV		 3479static void gen6_enable_rps_interrupts(struct drm_device *dev)
3480{
3481 struct drm_i915_private *dev_priv = dev->dev_private;
3482
3483 spin_lock_irq(&dev_priv->irq_lock);
a0b3335a 3484 WARN_ON(dev_priv->rps.pm_iir);
a6706b45
D		 3485 snb_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
3486 I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events);
44fc7d5c 3487 spin_unlock_irq(&dev_priv->irq_lock);
44fc7d5c
DV		 3488}
3489
3280e8b0
BW		 3490static void parse_rp_state_cap(struct drm_i915_private *dev_priv, u32 rp_state_cap)
3491{
3492 /* All of these values are in units of 50MHz */
3493 dev_priv->rps.cur_freq = 0;
3494 /* static values from HW: RP0 < RPe < RP1 < RPn (min_freq) */
3495 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
3496 dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
3497 dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
3498 /* XXX: only BYT has a special efficient freq */
3499 dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
3500 /* hw_max = RP0 until we check for overclocking */
3501 dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
3502
3503 /* Preserve min/max settings in case of re-init */
3504 if (dev_priv->rps.max_freq_softlimit == 0)
3505 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
3506
3507 if (dev_priv->rps.min_freq_softlimit == 0)
3508 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
3509}
3510
6edee7f3
BW		 3511static void gen8_enable_rps(struct drm_device *dev)
3512{
3513 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 3514 struct intel_engine_cs *ring;
6edee7f3
BW		 3515 uint32_t rc6_mask = 0, rp_state_cap;
3516 int unused;
3517
3518 /* 1a: Software RC state - RC0 */
3519 I915_WRITE(GEN6_RC_STATE, 0);
3520
3521 /* 1c & 1d: Get forcewake during program sequence. Although the driver
3522 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
c8d9a590 3523 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
6edee7f3
BW		 3524
3525 /* 2a: Disable RC states. */
3526 I915_WRITE(GEN6_RC_CONTROL, 0);
3527
3528 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
3280e8b0 3529 parse_rp_state_cap(dev_priv, rp_state_cap);
6edee7f3
BW		 3530
3531 /* 2b: Program RC6 thresholds.*/
3532 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
3533 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
3534 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
3535 for_each_ring(ring, dev_priv, unused)
3536 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
3537 I915_WRITE(GEN6_RC_SLEEP, 0);
0d68b25e
TR		 3538 if (IS_BROADWELL(dev))
3539 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
3540 else
3541 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
6edee7f3
BW		 3542
3543 /* 3: Enable RC6 */
3544 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
3545 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
abbf9d2c 3546 intel_print_rc6_info(dev, rc6_mask);
0d68b25e
TR		 3547 if (IS_BROADWELL(dev))
3548 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
3549 GEN7_RC_CTL_TO_MODE |
3550 rc6_mask);
3551 else
3552 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
3553 GEN6_RC_CTL_EI_MODE(1) |
3554 rc6_mask);
6edee7f3
BW		 3555
3556 /* 4 Program defaults and thresholds for RPS*/
f9bdc585
BW		 3557 I915_WRITE(GEN6_RPNSWREQ,
3558 HSW_FREQUENCY(dev_priv->rps.rp1_freq));
3559 I915_WRITE(GEN6_RC_VIDEO_FREQ,
3560 HSW_FREQUENCY(dev_priv->rps.rp1_freq));
6edee7f3
BW		 3561 /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
3562 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
3563
3564 /* Docs recommend 900MHz, and 300 MHz respectively */
3565 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
b39fb297
BW		 3566 dev_priv->rps.max_freq_softlimit << 24 |
3567 dev_priv->rps.min_freq_softlimit << 16);
6edee7f3
BW		 3568
3569 I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
3570 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
3571 I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
3572 I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
3573
3574 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
3575
3576 /* 5: Enable RPS */
3577 I915_WRITE(GEN6_RP_CONTROL,
3578 GEN6_RP_MEDIA_TURBO |
3579 GEN6_RP_MEDIA_HW_NORMAL_MODE |
7405f42c 3580 GEN6_RP_MEDIA_IS_GFX |
6edee7f3
BW		 3581 GEN6_RP_ENABLE |
3582 GEN6_RP_UP_BUSY_AVG |
3583 GEN6_RP_DOWN_IDLE_AVG);
3584
3585 /* 6: Ring frequency + overclocking (our driver does this later */
3586
3587 gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8);
3588
0961021a 3589 gen8_enable_rps_interrupts(dev);
6edee7f3 3590
c8d9a590 3591 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
6edee7f3
BW		 3592}
3593
79f5b2c7 3594static void gen6_enable_rps(struct drm_device *dev)
2b4e57bd 3595{
79f5b2c7 3596 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 3597 struct intel_engine_cs *ring;
2a5913a8 3598 u32 rp_state_cap;
7b9e0ae6 3599 u32 gt_perf_status;
d060c169 3600 u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
2b4e57bd 3601 u32 gtfifodbg;
2b4e57bd 3602 int rc6_mode;
42c0526c 3603 int i, ret;
2b4e57bd 3604
4fc688ce 3605 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
79f5b2c7 3606
2b4e57bd
ED		 3607 /* Here begins a magic sequence of register writes to enable
3608 * auto-downclocking.
3609 *
3610 * Perhaps there might be some value in exposing these to
3611 * userspace...
3612 */
3613 I915_WRITE(GEN6_RC_STATE, 0);
2b4e57bd
ED		 3614
3615 /* Clear the DBG now so we don't confuse earlier errors */
3616 if ((gtfifodbg = I915_READ(GTFIFODBG))) {
3617 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
3618 I915_WRITE(GTFIFODBG, gtfifodbg);
3619 }
3620
c8d9a590 3621 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
2b4e57bd 3622
7b9e0ae6
CW		 3623 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
3624 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
3625
3280e8b0 3626 parse_rp_state_cap(dev_priv, rp_state_cap);
dd0a1aa1 3627
2b4e57bd
ED		 3628 /* disable the counters and set deterministic thresholds */
3629 I915_WRITE(GEN6_RC_CONTROL, 0);
3630
3631 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
3632 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
3633 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
3634 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
3635 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
3636
b4519513
CW		 3637 for_each_ring(ring, dev_priv, i)
3638 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
2b4e57bd
ED		 3639
3640 I915_WRITE(GEN6_RC_SLEEP, 0);
3641 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
29c78f60 3642 if (IS_IVYBRIDGE(dev))
351aa566
SM		 3643 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
3644 else
3645 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
0920a487 3646 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
2b4e57bd
ED		 3647 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
3648
5a7dc92a 3649 /* Check if we are enabling RC6 */
2b4e57bd
ED		 3650 rc6_mode = intel_enable_rc6(dev_priv->dev);
3651 if (rc6_mode & INTEL_RC6_ENABLE)
3652 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
3653
5a7dc92a
ED		 3654 /* We don't use those on Haswell */
3655 if (!IS_HASWELL(dev)) {
3656 if (rc6_mode & INTEL_RC6p_ENABLE)
3657 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
2b4e57bd 3658
5a7dc92a
ED		 3659 if (rc6_mode & INTEL_RC6pp_ENABLE)
3660 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
3661 }
2b4e57bd 3662
dc39fff7 3663 intel_print_rc6_info(dev, rc6_mask);
2b4e57bd
ED		 3664
3665 I915_WRITE(GEN6_RC_CONTROL,
3666 rc6_mask |
3667 GEN6_RC_CTL_EI_MODE(1) |
3668 GEN6_RC_CTL_HW_ENABLE);
3669
dd75fdc8
CW		 3670 /* Power down if completely idle for over 50ms */
3671 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
2b4e57bd 3672 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
2b4e57bd 3673
42c0526c 3674 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
d060c169 3675 if (ret)
42c0526c 3676 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
d060c169
BW		 3677
3678 ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
3679 if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
3680 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
b39fb297 3681 (dev_priv->rps.max_freq_softlimit & 0xff) * 50,
d060c169 3682 (pcu_mbox & 0xff) * 50);
b39fb297 3683 dev_priv->rps.max_freq = pcu_mbox & 0xff;
2b4e57bd
ED		 3684 }
3685
dd75fdc8 3686 dev_priv->rps.power = HIGH_POWER; /* force a reset */
b39fb297 3687 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
2b4e57bd 3688
44fc7d5c 3689 gen6_enable_rps_interrupts(dev);
2b4e57bd 3690
31643d54
BW		 3691 rc6vids = 0;
3692 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
3693 if (IS_GEN6(dev) && ret) {
3694 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
3695 } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
3696 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
3697 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
3698 rc6vids &= 0xffff00;
3699 rc6vids |= GEN6_ENCODE_RC6_VID(450);
3700 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
3701 if (ret)
3702 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
3703 }
3704
c8d9a590 3705 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
2b4e57bd
ED		 3706}
3707
c2bc2fc5 3708static void __gen6_update_ring_freq(struct drm_device *dev)
2b4e57bd 3709{
79f5b2c7 3710 struct drm_i915_private *dev_priv = dev->dev_private;
2b4e57bd 3711 int min_freq = 15;
3ebecd07
CW		 3712 unsigned int gpu_freq;
3713 unsigned int max_ia_freq, min_ring_freq;
2b4e57bd 3714 int scaling_factor = 180;
eda79642 3715 struct cpufreq_policy *policy;
2b4e57bd 3716
4fc688ce 3717 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
79f5b2c7 3718
eda79642
BW		 3719 policy = cpufreq_cpu_get(0);
3720 if (policy) {
3721 max_ia_freq = policy->cpuinfo.max_freq;
3722 cpufreq_cpu_put(policy);
3723 } else {
3724 /*
3725 * Default to measured freq if none found, PCU will ensure we
3726 * don't go over
3727 */
2b4e57bd 3728 max_ia_freq = tsc_khz;
eda79642 3729 }
2b4e57bd
ED		 3730
3731 /* Convert from kHz to MHz */
3732 max_ia_freq /= 1000;
3733
153b4b95 3734 min_ring_freq = I915_READ(DCLK) & 0xf;
f6aca45c
BW		 3735 /* convert DDR frequency from units of 266.6MHz to bandwidth */
3736 min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3ebecd07 3737
2b4e57bd
ED		 3738 /*
3739 * For each potential GPU frequency, load a ring frequency we'd like
3740 * to use for memory access. We do this by specifying the IA frequency
3741 * the PCU should use as a reference to determine the ring frequency.
3742 */
b39fb297 3743 for (gpu_freq = dev_priv->rps.max_freq_softlimit; gpu_freq >= dev_priv->rps.min_freq_softlimit;
2b4e57bd 3744 gpu_freq--) {
b39fb297 3745 int diff = dev_priv->rps.max_freq_softlimit - gpu_freq;
3ebecd07
CW		 3746 unsigned int ia_freq = 0, ring_freq = 0;
3747
46c764d4
BW		 3748 if (INTEL_INFO(dev)->gen >= 8) {
3749 /* max(2 * GT, DDR). NB: GT is 50MHz units */
3750 ring_freq = max(min_ring_freq, gpu_freq);
3751 } else if (IS_HASWELL(dev)) {
f6aca45c 3752 ring_freq = mult_frac(gpu_freq, 5, 4);
3ebecd07
CW		 3753 ring_freq = max(min_ring_freq, ring_freq);
3754 /* leave ia_freq as the default, chosen by cpufreq */
3755 } else {
3756 /* On older processors, there is no separate ring
3757 * clock domain, so in order to boost the bandwidth
3758 * of the ring, we need to upclock the CPU (ia_freq).
3759 *
3760 * For GPU frequencies less than 750MHz,
3761 * just use the lowest ring freq.
3762 */
3763 if (gpu_freq < min_freq)
3764 ia_freq = 800;
3765 else
3766 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
3767 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3768 }
2b4e57bd 3769
42c0526c
BW		 3770 sandybridge_pcode_write(dev_priv,
3771 GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3ebecd07
CW		 3772 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
3773 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
3774 gpu_freq);
2b4e57bd 3775 }
2b4e57bd
ED		 3776}
3777
c2bc2fc5
ID		 3778void gen6_update_ring_freq(struct drm_device *dev)
3779{
3780 struct drm_i915_private *dev_priv = dev->dev_private;
3781
3782 if (INTEL_INFO(dev)->gen < 6 || IS_VALLEYVIEW(dev))
3783 return;
3784
3785 mutex_lock(&dev_priv->rps.hw_lock);
3786 __gen6_update_ring_freq(dev);
3787 mutex_unlock(&dev_priv->rps.hw_lock);
3788}
3789
2b6b3a09
D		 3790int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
3791{
3792 u32 val, rp0;
3793
3794 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
3795 rp0 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) & PUNIT_GPU_STATUS_MAX_FREQ_MASK;
3796
3797 return rp0;
3798}
3799
3800static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
3801{
3802 u32 val, rpe;
3803
3804 val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
3805 rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
3806
3807 return rpe;
3808}
3809
3810int cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
3811{
3812 u32 val, rpn;
3813
3814 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
3815 rpn = (val >> PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT) & PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK;
3816 return rpn;
3817}
3818
0a073b84
JB		 3819int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
3820{
3821 u32 val, rp0;
3822
64936258 3823 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
0a073b84
JB		 3824
3825 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
3826 /* Clamp to max */
3827 rp0 = min_t(u32, rp0, 0xea);
3828
3829 return rp0;
3830}
3831
3832static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
3833{
3834 u32 val, rpe;
3835
64936258 3836 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
0a073b84 3837 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
64936258 3838 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
0a073b84
JB		 3839 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
3840
3841 return rpe;
3842}
3843
3844int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
3845{
64936258 3846 return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
0a073b84
JB		 3847}
3848
ae48434c
ID		 3849/* Check that the pctx buffer wasn't move under us. */
3850static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
3851{
3852 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
3853
3854 WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
3855 dev_priv->vlv_pctx->stolen->start);
3856}
3857
38807746
D		 3858
3859/* Check that the pcbr address is not empty. */
3860static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
3861{
3862 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
3863
3864 WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
3865}
3866
3867static void cherryview_setup_pctx(struct drm_device *dev)
3868{
3869 struct drm_i915_private *dev_priv = dev->dev_private;
3870 unsigned long pctx_paddr, paddr;
3871 struct i915_gtt *gtt = &dev_priv->gtt;
3872 u32 pcbr;
3873 int pctx_size = 32*1024;
3874
3875 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
3876
3877 pcbr = I915_READ(VLV_PCBR);
3878 if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
3879 paddr = (dev_priv->mm.stolen_base +
3880 (gtt->stolen_size - pctx_size));
3881
3882 pctx_paddr = (paddr & (~4095));
3883 I915_WRITE(VLV_PCBR, pctx_paddr);
3884 }
3885}
3886
c9cddffc
JB		 3887static void valleyview_setup_pctx(struct drm_device *dev)
3888{
3889 struct drm_i915_private *dev_priv = dev->dev_private;
3890 struct drm_i915_gem_object *pctx;
3891 unsigned long pctx_paddr;
3892 u32 pcbr;
3893 int pctx_size = 24*1024;
3894
17b0c1f7
ID		 3895 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
3896
c9cddffc
JB		 3897 pcbr = I915_READ(VLV_PCBR);
3898 if (pcbr) {
3899 /* BIOS set it up already, grab the pre-alloc'd space */
3900 int pcbr_offset;
3901
3902 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
3903 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
3904 pcbr_offset,
190d6cd5 3905 I915_GTT_OFFSET_NONE,
c9cddffc
JB		 3906 pctx_size);
3907 goto out;
3908 }
3909
3910 /*
3911 * From the Gunit register HAS:
3912 * The Gfx driver is expected to program this register and ensure
3913 * proper allocation within Gfx stolen memory. For example, this
3914 * register should be programmed such than the PCBR range does not
3915 * overlap with other ranges, such as the frame buffer, protected
3916 * memory, or any other relevant ranges.
3917 */
3918 pctx = i915_gem_object_create_stolen(dev, pctx_size);
3919 if (!pctx) {
3920 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
3921 return;
3922 }
3923
3924 pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
3925 I915_WRITE(VLV_PCBR, pctx_paddr);
3926
3927out:
3928 dev_priv->vlv_pctx = pctx;
3929}
3930
ae48434c
ID		 3931static void valleyview_cleanup_pctx(struct drm_device *dev)
3932{
3933 struct drm_i915_private *dev_priv = dev->dev_private;
3934
3935 if (WARN_ON(!dev_priv->vlv_pctx))
3936 return;
3937
3938 drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
3939 dev_priv->vlv_pctx = NULL;
3940}
3941
4e80519e
ID		 3942static void valleyview_init_gt_powersave(struct drm_device *dev)
3943{
3944 struct drm_i915_private *dev_priv = dev->dev_private;
3945
3946 valleyview_setup_pctx(dev);
3947
3948 mutex_lock(&dev_priv->rps.hw_lock);
3949
3950 dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
3951 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
3952 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
3953 vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq),
3954 dev_priv->rps.max_freq);
3955
3956 dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
3957 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
3958 vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
3959 dev_priv->rps.efficient_freq);
3960
3961 dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
3962 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
3963 vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq),
3964 dev_priv->rps.min_freq);
3965
3966 /* Preserve min/max settings in case of re-init */
3967 if (dev_priv->rps.max_freq_softlimit == 0)
3968 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
3969
3970 if (dev_priv->rps.min_freq_softlimit == 0)
3971 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
3972
3973 mutex_unlock(&dev_priv->rps.hw_lock);
3974}
3975
38807746
D		 3976static void cherryview_init_gt_powersave(struct drm_device *dev)
3977{
2b6b3a09
D		 3978 struct drm_i915_private *dev_priv = dev->dev_private;
3979
38807746 3980 cherryview_setup_pctx(dev);
2b6b3a09
D		 3981
3982 mutex_lock(&dev_priv->rps.hw_lock);
3983
3984 dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
3985 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
3986 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
3987 vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq),
3988 dev_priv->rps.max_freq);
3989
3990 dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
3991 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
3992 vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
3993 dev_priv->rps.efficient_freq);
3994
3995 dev_priv->rps.min_freq = cherryview_rps_min_freq(dev_priv);
3996 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
3997 vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq),
3998 dev_priv->rps.min_freq);
3999
4000 /* Preserve min/max settings in case of re-init */
4001 if (dev_priv->rps.max_freq_softlimit == 0)
4002 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4003
4004 if (dev_priv->rps.min_freq_softlimit == 0)
4005 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
4006
4007 mutex_unlock(&dev_priv->rps.hw_lock);
38807746
D		 4008}
4009
4e80519e
ID		 4010static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
4011{
4012 valleyview_cleanup_pctx(dev);
4013}
4014
38807746
D		 4015static void cherryview_enable_rps(struct drm_device *dev)
4016{
4017 struct drm_i915_private *dev_priv = dev->dev_private;
4018 struct intel_engine_cs *ring;
2b6b3a09 4019 u32 gtfifodbg, val, rc6_mode = 0, pcbr;
38807746
D		 4020 int i;
4021
4022 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4023
4024 gtfifodbg = I915_READ(GTFIFODBG);
4025 if (gtfifodbg) {
4026 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
4027 gtfifodbg);
4028 I915_WRITE(GTFIFODBG, gtfifodbg);
4029 }
4030
4031 cherryview_check_pctx(dev_priv);
4032
4033 /* 1a & 1b: Get forcewake during program sequence. Although the driver
4034 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
4035 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
4036
4037 /* 2a: Program RC6 thresholds.*/
4038 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
4039 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4040 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4041
4042 for_each_ring(ring, dev_priv, i)
4043 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4044 I915_WRITE(GEN6_RC_SLEEP, 0);
4045
4046 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
4047
4048 /* allows RC6 residency counter to work */
4049 I915_WRITE(VLV_COUNTER_CONTROL,
4050 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
4051 VLV_MEDIA_RC6_COUNT_EN |
4052 VLV_RENDER_RC6_COUNT_EN));
4053
4054 /* For now we assume BIOS is allocating and populating the PCBR */
4055 pcbr = I915_READ(VLV_PCBR);
4056
4057 DRM_DEBUG_DRIVER("PCBR offset : 0x%x\n", pcbr);
4058
4059 /* 3: Enable RC6 */
4060 if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
4061 (pcbr >> VLV_PCBR_ADDR_SHIFT))
4062 rc6_mode = GEN6_RC_CTL_EI_MODE(1);
4063
4064 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
4065
2b6b3a09
D		 4066 /* 4 Program defaults and thresholds for RPS*/
4067 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
4068 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
4069 I915_WRITE(GEN6_RP_UP_EI, 66000);
4070 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
4071
4072 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4073
7405f42c
TR		 4074 /* WaDisablePwrmtrEvent:chv (pre-production hw) */
4075 I915_WRITE(0xA80C, I915_READ(0xA80C) & 0x00ffffff);
4076 I915_WRITE(0xA810, I915_READ(0xA810) & 0xffffff00);
4077
2b6b3a09
D		 4078 /* 5: Enable RPS */
4079 I915_WRITE(GEN6_RP_CONTROL,
4080 GEN6_RP_MEDIA_HW_NORMAL_MODE |
7405f42c 4081 GEN6_RP_MEDIA_IS_GFX | /* WaSetMaskForGfxBusyness:chv (pre-production hw ?) */
2b6b3a09
D		 4082 GEN6_RP_ENABLE |
4083 GEN6_RP_UP_BUSY_AVG |
4084 GEN6_RP_DOWN_IDLE_AVG);
4085
4086 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4087
4088 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
4089 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
4090
4091 dev_priv->rps.cur_freq = (val >> 8) & 0xff;
4092 DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
4093 vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
4094 dev_priv->rps.cur_freq);
4095
4096 DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
4097 vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
4098 dev_priv->rps.efficient_freq);
4099
4100 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
4101
38807746
D		 4102 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
4103}
4104
0a073b84
JB		 4105static void valleyview_enable_rps(struct drm_device *dev)
4106{
4107 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 4108 struct intel_engine_cs *ring;
2a5913a8 4109 u32 gtfifodbg, val, rc6_mode = 0;
0a073b84
JB		 4110 int i;
4111
4112 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4113
ae48434c
ID		 4114 valleyview_check_pctx(dev_priv);
4115
0a073b84 4116 if ((gtfifodbg = I915_READ(GTFIFODBG))) {
f7d85c1e
JB		 4117 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
4118 gtfifodbg);
0a073b84
JB		 4119 I915_WRITE(GTFIFODBG, gtfifodbg);
4120 }
4121
c8d9a590
D		 4122 /* If VLV, Forcewake all wells, else re-direct to regular path */
4123 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
0a073b84
JB		 4124
4125 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
4126 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
4127 I915_WRITE(GEN6_RP_UP_EI, 66000);
4128 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
4129
4130 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
31685c25 4131 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 0xf4240);
0a073b84
JB		 4132
4133 I915_WRITE(GEN6_RP_CONTROL,
4134 GEN6_RP_MEDIA_TURBO |
4135 GEN6_RP_MEDIA_HW_NORMAL_MODE |
4136 GEN6_RP_MEDIA_IS_GFX |
4137 GEN6_RP_ENABLE |
4138 GEN6_RP_UP_BUSY_AVG |
4139 GEN6_RP_DOWN_IDLE_CONT);
4140
4141 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
4142 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
4143 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
4144
4145 for_each_ring(ring, dev_priv, i)
4146 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4147
2f0aa304 4148 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
0a073b84
JB		 4149
4150 /* allows RC6 residency counter to work */
49798eb2 4151 I915_WRITE(VLV_COUNTER_CONTROL,
31685c25
D		 4152 _MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN |
4153 VLV_RENDER_RC0_COUNT_EN |
49798eb2
JB		 4154 VLV_MEDIA_RC6_COUNT_EN |
4155 VLV_RENDER_RC6_COUNT_EN));
31685c25 4156
a2b23fe0 4157 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
6b88f295 4158 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
dc39fff7
BW		 4159
4160 intel_print_rc6_info(dev, rc6_mode);
4161
a2b23fe0 4162 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
0a073b84 4163
64936258 4164 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
0a073b84
JB		 4165
4166 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
4167 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
4168
b39fb297 4169 dev_priv->rps.cur_freq = (val >> 8) & 0xff;
73008b98 4170 DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
b39fb297
BW		 4171 vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
4172 dev_priv->rps.cur_freq);
0a073b84 4173
73008b98 4174 DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
b39fb297
BW		 4175 vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
4176 dev_priv->rps.efficient_freq);
0a073b84 4177
b39fb297 4178 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
0a073b84 4179
44fc7d5c 4180 gen6_enable_rps_interrupts(dev);
0a073b84 4181
c8d9a590 4182 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
0a073b84
JB		 4183}
4184
930ebb46 4185void ironlake_teardown_rc6(struct drm_device *dev)
2b4e57bd
ED		 4186{
4187 struct drm_i915_private *dev_priv = dev->dev_private;
4188
3e373948 4189 if (dev_priv->ips.renderctx) {
d7f46fc4 4190 i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx);
3e373948
DV		 4191 drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
4192 dev_priv->ips.renderctx = NULL;
2b4e57bd
ED		 4193 }
4194
3e373948 4195 if (dev_priv->ips.pwrctx) {
d7f46fc4 4196 i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx);
3e373948
DV		 4197 drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
4198 dev_priv->ips.pwrctx = NULL;
2b4e57bd
ED		 4199 }
4200}
4201
930ebb46 4202static void ironlake_disable_rc6(struct drm_device *dev)
2b4e57bd
ED		 4203{
4204 struct drm_i915_private *dev_priv = dev->dev_private;
4205
4206 if (I915_READ(PWRCTXA)) {
4207 /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
4208 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT);
4209 wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
4210 50);
4211
4212 I915_WRITE(PWRCTXA, 0);
4213 POSTING_READ(PWRCTXA);
4214
4215 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
4216 POSTING_READ(RSTDBYCTL);
4217 }
2b4e57bd
ED		 4218}
4219
4220static int ironlake_setup_rc6(struct drm_device *dev)
4221{
4222 struct drm_i915_private *dev_priv = dev->dev_private;
4223
3e373948
DV		 4224 if (dev_priv->ips.renderctx == NULL)
4225 dev_priv->ips.renderctx = intel_alloc_context_page(dev);
4226 if (!dev_priv->ips.renderctx)
2b4e57bd
ED		 4227 return -ENOMEM;
4228
3e373948
DV		 4229 if (dev_priv->ips.pwrctx == NULL)
4230 dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
4231 if (!dev_priv->ips.pwrctx) {
2b4e57bd
ED		 4232 ironlake_teardown_rc6(dev);
4233 return -ENOMEM;
4234 }
4235
4236 return 0;
4237}
4238
930ebb46 4239static void ironlake_enable_rc6(struct drm_device *dev)
2b4e57bd
ED		 4240{
4241 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 4242 struct intel_engine_cs *ring = &dev_priv->ring[RCS];
3e960501 4243 bool was_interruptible;
2b4e57bd
ED		 4244 int ret;
4245
4246 /* rc6 disabled by default due to repeated reports of hanging during
4247 * boot and resume.
4248 */
4249 if (!intel_enable_rc6(dev))
4250 return;
4251
79f5b2c7
DV		 4252 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
4253
2b4e57bd 4254 ret = ironlake_setup_rc6(dev);
79f5b2c7 4255 if (ret)
2b4e57bd 4256 return;
2b4e57bd 4257
3e960501
CW		 4258 was_interruptible = dev_priv->mm.interruptible;
4259 dev_priv->mm.interruptible = false;
4260
2b4e57bd
ED		 4261 /*
4262 * GPU can automatically power down the render unit if given a page
4263 * to save state.
4264 */
6d90c952 4265 ret = intel_ring_begin(ring, 6);
2b4e57bd
ED		 4266 if (ret) {
4267 ironlake_teardown_rc6(dev);
3e960501 4268 dev_priv->mm.interruptible = was_interruptible;
2b4e57bd
ED		 4269 return;
4270 }
4271
6d90c952
DV		 4272 intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
4273 intel_ring_emit(ring, MI_SET_CONTEXT);
f343c5f6 4274 intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) |
6d90c952
DV		 4275 MI_MM_SPACE_GTT |
4276 MI_SAVE_EXT_STATE_EN |
4277 MI_RESTORE_EXT_STATE_EN |
4278 MI_RESTORE_INHIBIT);
4279 intel_ring_emit(ring, MI_SUSPEND_FLUSH);
4280 intel_ring_emit(ring, MI_NOOP);
4281 intel_ring_emit(ring, MI_FLUSH);
4282 intel_ring_advance(ring);
2b4e57bd
ED		 4283
4284 /*
4285 * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
4286 * does an implicit flush, combined with MI_FLUSH above, it should be
4287 * safe to assume that renderctx is valid
4288 */
3e960501
CW		 4289 ret = intel_ring_idle(ring);
4290 dev_priv->mm.interruptible = was_interruptible;
2b4e57bd 4291 if (ret) {
def27a58 4292 DRM_ERROR("failed to enable ironlake power savings\n");
2b4e57bd 4293 ironlake_teardown_rc6(dev);
2b4e57bd
ED		 4294 return;
4295 }
4296
f343c5f6 4297 I915_WRITE(PWRCTXA, i915_gem_obj_ggtt_offset(dev_priv->ips.pwrctx) | PWRCTX_EN);
2b4e57bd 4298 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
dc39fff7 4299
91ca689a 4300 intel_print_rc6_info(dev, GEN6_RC_CTL_RC6_ENABLE);
2b4e57bd
ED		 4301}
4302
dde18883
ED		 4303static unsigned long intel_pxfreq(u32 vidfreq)
4304{
4305 unsigned long freq;
4306 int div = (vidfreq & 0x3f0000) >> 16;
4307 int post = (vidfreq & 0x3000) >> 12;
4308 int pre = (vidfreq & 0x7);
4309
4310 if (!pre)
4311 return 0;
4312
4313 freq = ((div * 133333) / ((1<<post) * pre));
4314
4315 return freq;
4316}
4317
eb48eb00
DV		 4318static const struct cparams {
4319 u16 i;
4320 u16 t;
4321 u16 m;
4322 u16 c;
4323} cparams[] = {
4324 { 1, 1333, 301, 28664 },
4325 { 1, 1066, 294, 24460 },
4326 { 1, 800, 294, 25192 },
4327 { 0, 1333, 276, 27605 },
4328 { 0, 1066, 276, 27605 },
4329 { 0, 800, 231, 23784 },
4330};
4331
f531dcb2 4332static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
eb48eb00
DV		 4333{
4334 u64 total_count, diff, ret;
4335 u32 count1, count2, count3, m = 0, c = 0;
4336 unsigned long now = jiffies_to_msecs(jiffies), diff1;
4337 int i;
4338
02d71956
DV		 4339 assert_spin_locked(&mchdev_lock);
4340
20e4d407 4341 diff1 = now - dev_priv->ips.last_time1;
eb48eb00
DV		 4342
4343 /* Prevent division-by-zero if we are asking too fast.
4344 * Also, we don't get interesting results if we are polling
4345 * faster than once in 10ms, so just return the saved value
4346 * in such cases.
4347 */
4348 if (diff1 <= 10)
20e4d407 4349 return dev_priv->ips.chipset_power;
eb48eb00
DV		 4350
4351 count1 = I915_READ(DMIEC);
4352 count2 = I915_READ(DDREC);
4353 count3 = I915_READ(CSIEC);
4354
4355 total_count = count1 + count2 + count3;
4356
4357 /* FIXME: handle per-counter overflow */
20e4d407
DV		 4358 if (total_count < dev_priv->ips.last_count1) {
4359 diff = ~0UL - dev_priv->ips.last_count1;
eb48eb00
DV		 4360 diff += total_count;
4361 } else {
20e4d407 4362 diff = total_count - dev_priv->ips.last_count1;
eb48eb00
DV		 4363 }
4364
4365 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
20e4d407
DV		 4366 if (cparams[i].i == dev_priv->ips.c_m &&
4367 cparams[i].t == dev_priv->ips.r_t) {
eb48eb00
DV		 4368 m = cparams[i].m;
4369 c = cparams[i].c;
4370 break;
4371 }
4372 }
4373
4374 diff = div_u64(diff, diff1);
4375 ret = ((m * diff) + c);
4376 ret = div_u64(ret, 10);
4377
20e4d407
DV		 4378 dev_priv->ips.last_count1 = total_count;
4379 dev_priv->ips.last_time1 = now;
eb48eb00 4380
20e4d407 4381 dev_priv->ips.chipset_power = ret;
eb48eb00
DV		 4382
4383 return ret;
4384}
4385
f531dcb2
CW		 4386unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
4387{
3d13ef2e 4388 struct drm_device *dev = dev_priv->dev;
f531dcb2
CW		 4389 unsigned long val;
4390
3d13ef2e 4391 if (INTEL_INFO(dev)->gen != 5)
f531dcb2
CW		 4392 return 0;
4393
4394 spin_lock_irq(&mchdev_lock);
4395
4396 val = __i915_chipset_val(dev_priv);
4397
4398 spin_unlock_irq(&mchdev_lock);
4399
4400 return val;
4401}
4402
eb48eb00
DV		 4403unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
4404{
4405 unsigned long m, x, b;
4406 u32 tsfs;
4407
4408 tsfs = I915_READ(TSFS);
4409
4410 m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
4411 x = I915_READ8(TR1);
4412
4413 b = tsfs & TSFS_INTR_MASK;
4414
4415 return ((m * x) / 127) - b;
4416}
4417
4418static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
4419{
3d13ef2e 4420 struct drm_device *dev = dev_priv->dev;
eb48eb00
DV		 4421 static const struct v_table {
4422 u16 vd; /* in .1 mil */
4423 u16 vm; /* in .1 mil */
4424 } v_table[] = {
4425 { 0, 0, },
4426 { 375, 0, },
4427 { 500, 0, },
4428 { 625, 0, },
4429 { 750, 0, },
4430 { 875, 0, },
4431 { 1000, 0, },
4432 { 1125, 0, },
4433 { 4125, 3000, },
4434 { 4125, 3000, },
4435 { 4125, 3000, },
4436 { 4125, 3000, },
4437 { 4125, 3000, },
4438 { 4125, 3000, },
4439 { 4125, 3000, },
4440 { 4125, 3000, },
4441 { 4125, 3000, },
4442 { 4125, 3000, },
4443 { 4125, 3000, },
4444 { 4125, 3000, },
4445 { 4125, 3000, },
4446 { 4125, 3000, },
4447 { 4125, 3000, },
4448 { 4125, 3000, },
4449 { 4125, 3000, },
4450 { 4125, 3000, },
4451 { 4125, 3000, },
4452 { 4125, 3000, },
4453 { 4125, 3000, },
4454 { 4125, 3000, },
4455 { 4125, 3000, },
4456 { 4125, 3000, },
4457 { 4250, 3125, },
4458 { 4375, 3250, },
4459 { 4500, 3375, },
4460 { 4625, 3500, },
4461 { 4750, 3625, },
4462 { 4875, 3750, },
4463 { 5000, 3875, },
4464 { 5125, 4000, },
4465 { 5250, 4125, },
4466 { 5375, 4250, },
4467 { 5500, 4375, },
4468 { 5625, 4500, },
4469 { 5750, 4625, },
4470 { 5875, 4750, },
4471 { 6000, 4875, },
4472 { 6125, 5000, },
4473 { 6250, 5125, },
4474 { 6375, 5250, },
4475 { 6500, 5375, },
4476 { 6625, 5500, },
4477 { 6750, 5625, },
4478 { 6875, 5750, },
4479 { 7000, 5875, },
4480 { 7125, 6000, },
4481 { 7250, 6125, },
4482 { 7375, 6250, },
4483 { 7500, 6375, },
4484 { 7625, 6500, },
4485 { 7750, 6625, },
4486 { 7875, 6750, },
4487 { 8000, 6875, },
4488 { 8125, 7000, },
4489 { 8250, 7125, },
4490 { 8375, 7250, },
4491 { 8500, 7375, },
4492 { 8625, 7500, },
4493 { 8750, 7625, },
4494 { 8875, 7750, },
4495 { 9000, 7875, },
4496 { 9125, 8000, },
4497 { 9250, 8125, },
4498 { 9375, 8250, },
4499 { 9500, 8375, },
4500 { 9625, 8500, },
4501 { 9750, 8625, },
4502 { 9875, 8750, },
4503 { 10000, 8875, },
4504 { 10125, 9000, },
4505 { 10250, 9125, },
4506 { 10375, 9250, },
4507 { 10500, 9375, },
4508 { 10625, 9500, },
4509 { 10750, 9625, },
4510 { 10875, 9750, },
4511 { 11000, 9875, },
4512 { 11125, 10000, },
4513 { 11250, 10125, },
4514 { 11375, 10250, },
4515 { 11500, 10375, },
4516 { 11625, 10500, },
4517 { 11750, 10625, },
4518 { 11875, 10750, },
4519 { 12000, 10875, },
4520 { 12125, 11000, },
4521 { 12250, 11125, },
4522 { 12375, 11250, },
4523 { 12500, 11375, },
4524 { 12625, 11500, },
4525 { 12750, 11625, },
4526 { 12875, 11750, },
4527 { 13000, 11875, },
4528 { 13125, 12000, },
4529 { 13250, 12125, },
4530 { 13375, 12250, },
4531 { 13500, 12375, },
4532 { 13625, 12500, },
4533 { 13750, 12625, },
4534 { 13875, 12750, },
4535 { 14000, 12875, },
4536 { 14125, 13000, },
4537 { 14250, 13125, },
4538 { 14375, 13250, },
4539 { 14500, 13375, },
4540 { 14625, 13500, },
4541 { 14750, 13625, },
4542 { 14875, 13750, },
4543 { 15000, 13875, },
4544 { 15125, 14000, },
4545 { 15250, 14125, },
4546 { 15375, 14250, },
4547 { 15500, 14375, },
4548 { 15625, 14500, },
4549 { 15750, 14625, },
4550 { 15875, 14750, },
4551 { 16000, 14875, },
4552 { 16125, 15000, },
4553 };
3d13ef2e 4554 if (INTEL_INFO(dev)->is_mobile)
eb48eb00
DV		 4555 return v_table[pxvid].vm;
4556 else
4557 return v_table[pxvid].vd;
4558}
4559
02d71956 4560static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
eb48eb00
DV		 4561{
4562 struct timespec now, diff1;
4563 u64 diff;
4564 unsigned long diffms;
4565 u32 count;
4566
02d71956 4567 assert_spin_locked(&mchdev_lock);
eb48eb00
DV		 4568
4569 getrawmonotonic(&now);
20e4d407 4570 diff1 = timespec_sub(now, dev_priv->ips.last_time2);
eb48eb00
DV		 4571
4572 /* Don't divide by 0 */
4573 diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
4574 if (!diffms)
4575 return;
4576
4577 count = I915_READ(GFXEC);
4578
20e4d407
DV		 4579 if (count < dev_priv->ips.last_count2) {
4580 diff = ~0UL - dev_priv->ips.last_count2;
eb48eb00
DV		 4581 diff += count;
4582 } else {
20e4d407 4583 diff = count - dev_priv->ips.last_count2;
eb48eb00
DV		 4584 }
4585
20e4d407
DV		 4586 dev_priv->ips.last_count2 = count;
4587 dev_priv->ips.last_time2 = now;
eb48eb00
DV		 4588
4589 /* More magic constants... */
4590 diff = diff * 1181;
4591 diff = div_u64(diff, diffms * 10);
20e4d407 4592 dev_priv->ips.gfx_power = diff;
eb48eb00
DV		 4593}
4594
02d71956
DV		 4595void i915_update_gfx_val(struct drm_i915_private *dev_priv)
4596{
3d13ef2e
DL		 4597 struct drm_device *dev = dev_priv->dev;
4598
4599 if (INTEL_INFO(dev)->gen != 5)
02d71956
DV		 4600 return;
4601
9270388e 4602 spin_lock_irq(&mchdev_lock);
02d71956
DV		 4603
4604 __i915_update_gfx_val(dev_priv);
4605
9270388e 4606 spin_unlock_irq(&mchdev_lock);
02d71956
DV		 4607}
4608
f531dcb2 4609static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
eb48eb00
DV		 4610{
4611 unsigned long t, corr, state1, corr2, state2;
4612 u32 pxvid, ext_v;
4613
02d71956
DV		 4614 assert_spin_locked(&mchdev_lock);
4615
b39fb297 4616 pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4));
eb48eb00
DV		 4617 pxvid = (pxvid >> 24) & 0x7f;
4618 ext_v = pvid_to_extvid(dev_priv, pxvid);
4619
4620 state1 = ext_v;
4621
4622 t = i915_mch_val(dev_priv);
4623
4624 /* Revel in the empirically derived constants */
4625
4626 /* Correction factor in 1/100000 units */
4627 if (t > 80)
4628 corr = ((t * 2349) + 135940);
4629 else if (t >= 50)
4630 corr = ((t * 964) + 29317);
4631 else /* < 50 */
4632 corr = ((t * 301) + 1004);
4633
4634 corr = corr * ((150142 * state1) / 10000 - 78642);
4635 corr /= 100000;
20e4d407 4636 corr2 = (corr * dev_priv->ips.corr);
eb48eb00
DV		 4637
4638 state2 = (corr2 * state1) / 10000;
4639 state2 /= 100; /* convert to mW */
4640
02d71956 4641 __i915_update_gfx_val(dev_priv);
eb48eb00 4642
20e4d407 4643 return dev_priv->ips.gfx_power + state2;
eb48eb00
DV		 4644}
4645
f531dcb2
CW		 4646unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
4647{
3d13ef2e 4648 struct drm_device *dev = dev_priv->dev;
f531dcb2
CW		 4649 unsigned long val;
4650
3d13ef2e 4651 if (INTEL_INFO(dev)->gen != 5)
f531dcb2
CW		 4652 return 0;
4653
4654 spin_lock_irq(&mchdev_lock);
4655
4656 val = __i915_gfx_val(dev_priv);
4657
4658 spin_unlock_irq(&mchdev_lock);
4659
4660 return val;
4661}
4662
eb48eb00
DV		 4663/**
4664 * i915_read_mch_val - return value for IPS use
4665 *
4666 * Calculate and return a value for the IPS driver to use when deciding whether
4667 * we have thermal and power headroom to increase CPU or GPU power budget.
4668 */
4669unsigned long i915_read_mch_val(void)
4670{
4671 struct drm_i915_private *dev_priv;
4672 unsigned long chipset_val, graphics_val, ret = 0;
4673
9270388e 4674 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 4675 if (!i915_mch_dev)
4676 goto out_unlock;
4677 dev_priv = i915_mch_dev;
4678
f531dcb2
CW		 4679 chipset_val = __i915_chipset_val(dev_priv);
4680 graphics_val = __i915_gfx_val(dev_priv);
eb48eb00
DV		 4681
4682 ret = chipset_val + graphics_val;
4683
4684out_unlock:
9270388e 4685 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 4686
4687 return ret;
4688}
4689EXPORT_SYMBOL_GPL(i915_read_mch_val);
4690
4691/**
4692 * i915_gpu_raise - raise GPU frequency limit
4693 *
4694 * Raise the limit; IPS indicates we have thermal headroom.
4695 */
4696bool i915_gpu_raise(void)
4697{
4698 struct drm_i915_private *dev_priv;
4699 bool ret = true;
4700
9270388e 4701 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 4702 if (!i915_mch_dev) {
4703 ret = false;
4704 goto out_unlock;
4705 }
4706 dev_priv = i915_mch_dev;
4707
20e4d407
DV		 4708 if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
4709 dev_priv->ips.max_delay--;
eb48eb00
DV		 4710
4711out_unlock:
9270388e 4712 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 4713
4714 return ret;
4715}
4716EXPORT_SYMBOL_GPL(i915_gpu_raise);
4717
4718/**
4719 * i915_gpu_lower - lower GPU frequency limit
4720 *
4721 * IPS indicates we're close to a thermal limit, so throttle back the GPU
4722 * frequency maximum.
4723 */
4724bool i915_gpu_lower(void)
4725{
4726 struct drm_i915_private *dev_priv;
4727 bool ret = true;
4728
9270388e 4729 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 4730 if (!i915_mch_dev) {
4731 ret = false;
4732 goto out_unlock;
4733 }
4734 dev_priv = i915_mch_dev;
4735
20e4d407
DV		 4736 if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
4737 dev_priv->ips.max_delay++;
eb48eb00
DV		 4738
4739out_unlock:
9270388e 4740 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 4741
4742 return ret;
4743}
4744EXPORT_SYMBOL_GPL(i915_gpu_lower);
4745
4746/**
4747 * i915_gpu_busy - indicate GPU business to IPS
4748 *
4749 * Tell the IPS driver whether or not the GPU is busy.
4750 */
4751bool i915_gpu_busy(void)
4752{
4753 struct drm_i915_private *dev_priv;
a4872ba6 4754 struct intel_engine_cs *ring;
eb48eb00 4755 bool ret = false;
f047e395 4756 int i;
eb48eb00 4757
9270388e 4758 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 4759 if (!i915_mch_dev)
4760 goto out_unlock;
4761 dev_priv = i915_mch_dev;
4762
f047e395
CW		 4763 for_each_ring(ring, dev_priv, i)
4764 ret |= !list_empty(&ring->request_list);
eb48eb00
DV		 4765
4766out_unlock:
9270388e 4767 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 4768
4769 return ret;
4770}
4771EXPORT_SYMBOL_GPL(i915_gpu_busy);
4772
4773/**
4774 * i915_gpu_turbo_disable - disable graphics turbo
4775 *
4776 * Disable graphics turbo by resetting the max frequency and setting the
4777 * current frequency to the default.
4778 */
4779bool i915_gpu_turbo_disable(void)
4780{
4781 struct drm_i915_private *dev_priv;
4782 bool ret = true;
4783
9270388e 4784 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 4785 if (!i915_mch_dev) {
4786 ret = false;
4787 goto out_unlock;
4788 }
4789 dev_priv = i915_mch_dev;
4790
20e4d407 4791 dev_priv->ips.max_delay = dev_priv->ips.fstart;
eb48eb00 4792
20e4d407 4793 if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
eb48eb00
DV		 4794 ret = false;
4795
4796out_unlock:
9270388e 4797 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 4798
4799 return ret;
4800}
4801EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
4802
4803/**
4804 * Tells the intel_ips driver that the i915 driver is now loaded, if
4805 * IPS got loaded first.
4806 *
4807 * This awkward dance is so that neither module has to depend on the
4808 * other in order for IPS to do the appropriate communication of
4809 * GPU turbo limits to i915.
4810 */
4811static void
4812ips_ping_for_i915_load(void)
4813{
4814 void (*link)(void);
4815
4816 link = symbol_get(ips_link_to_i915_driver);
4817 if (link) {
4818 link();
4819 symbol_put(ips_link_to_i915_driver);
4820 }
4821}
4822
4823void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
4824{
02d71956
DV		 4825 /* We only register the i915 ips part with intel-ips once everything is
4826 * set up, to avoid intel-ips sneaking in and reading bogus values. */
9270388e 4827 spin_lock_irq(&mchdev_lock);
eb48eb00 4828 i915_mch_dev = dev_priv;
9270388e 4829 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 4830
4831 ips_ping_for_i915_load();
4832}
4833
4834void intel_gpu_ips_teardown(void)
4835{
9270388e 4836 spin_lock_irq(&mchdev_lock);
eb48eb00 4837 i915_mch_dev = NULL;
9270388e 4838 spin_unlock_irq(&mchdev_lock);
eb48eb00 4839}
76c3552f 4840
8090c6b9 4841static void intel_init_emon(struct drm_device *dev)
dde18883
ED		 4842{
4843 struct drm_i915_private *dev_priv = dev->dev_private;
4844 u32 lcfuse;
4845 u8 pxw[16];
4846 int i;
4847
4848 /* Disable to program */
4849 I915_WRITE(ECR, 0);
4850 POSTING_READ(ECR);
4851
4852 /* Program energy weights for various events */
4853 I915_WRITE(SDEW, 0x15040d00);
4854 I915_WRITE(CSIEW0, 0x007f0000);
4855 I915_WRITE(CSIEW1, 0x1e220004);
4856 I915_WRITE(CSIEW2, 0x04000004);
4857
4858 for (i = 0; i < 5; i++)
4859 I915_WRITE(PEW + (i * 4), 0);
4860 for (i = 0; i < 3; i++)
4861 I915_WRITE(DEW + (i * 4), 0);
4862
4863 /* Program P-state weights to account for frequency power adjustment */
4864 for (i = 0; i < 16; i++) {
4865 u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
4866 unsigned long freq = intel_pxfreq(pxvidfreq);
4867 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
4868 PXVFREQ_PX_SHIFT;
4869 unsigned long val;
4870
4871 val = vid * vid;
4872 val *= (freq / 1000);
4873 val *= 255;
4874 val /= (127*127*900);
4875 if (val > 0xff)
4876 DRM_ERROR("bad pxval: %ld\n", val);
4877 pxw[i] = val;
4878 }
4879 /* Render standby states get 0 weight */
4880 pxw[14] = 0;
4881 pxw[15] = 0;
4882
4883 for (i = 0; i < 4; i++) {
4884 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
4885 (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
4886 I915_WRITE(PXW + (i * 4), val);
4887 }
4888
4889 /* Adjust magic regs to magic values (more experimental results) */
4890 I915_WRITE(OGW0, 0);
4891 I915_WRITE(OGW1, 0);
4892 I915_WRITE(EG0, 0x00007f00);
4893 I915_WRITE(EG1, 0x0000000e);
4894 I915_WRITE(EG2, 0x000e0000);
4895 I915_WRITE(EG3, 0x68000300);
4896 I915_WRITE(EG4, 0x42000000);
4897 I915_WRITE(EG5, 0x00140031);
4898 I915_WRITE(EG6, 0);
4899 I915_WRITE(EG7, 0);
4900
4901 for (i = 0; i < 8; i++)
4902 I915_WRITE(PXWL + (i * 4), 0);
4903
4904 /* Enable PMON + select events */
4905 I915_WRITE(ECR, 0x80000019);
4906
4907 lcfuse = I915_READ(LCFUSE02);
4908
20e4d407 4909 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
dde18883
ED		 4910}
4911
ae48434c
ID		 4912void intel_init_gt_powersave(struct drm_device *dev)
4913{
e6069ca8
ID		 4914 i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
4915
38807746
D		 4916 if (IS_CHERRYVIEW(dev))
4917 cherryview_init_gt_powersave(dev);
4918 else if (IS_VALLEYVIEW(dev))
4e80519e 4919 valleyview_init_gt_powersave(dev);
ae48434c
ID		 4920}
4921
4922void intel_cleanup_gt_powersave(struct drm_device *dev)
4923{
38807746
D		 4924 if (IS_CHERRYVIEW(dev))
4925 return;
4926 else if (IS_VALLEYVIEW(dev))
4e80519e 4927 valleyview_cleanup_gt_powersave(dev);
ae48434c
ID		 4928}
4929
156c7ca0
JB		 4930/**
4931 * intel_suspend_gt_powersave - suspend PM work and helper threads
4932 * @dev: drm device
4933 *
4934 * We don't want to disable RC6 or other features here, we just want
4935 * to make sure any work we've queued has finished and won't bother
4936 * us while we're suspended.
4937 */
4938void intel_suspend_gt_powersave(struct drm_device *dev)
4939{
4940 struct drm_i915_private *dev_priv = dev->dev_private;
4941
4942 /* Interrupts should be disabled already to avoid re-arming. */
e11aa362 4943 WARN_ON(dev->irq_enabled && !dev_priv->pm.irqs_disabled);
156c7ca0
JB		 4944
4945 flush_delayed_work(&dev_priv->rps.delayed_resume_work);
4946
4947 cancel_work_sync(&dev_priv->rps.work);
4948}
4949
8090c6b9
DV		 4950void intel_disable_gt_powersave(struct drm_device *dev)
4951{
1a01ab3b
JB		 4952 struct drm_i915_private *dev_priv = dev->dev_private;
4953
fd0c0642 4954 /* Interrupts should be disabled already to avoid re-arming. */
e11aa362 4955 WARN_ON(dev->irq_enabled && !dev_priv->pm.irqs_disabled);
fd0c0642 4956
930ebb46 4957 if (IS_IRONLAKE_M(dev)) {
8090c6b9 4958 ironlake_disable_drps(dev);
930ebb46 4959 ironlake_disable_rc6(dev);
38807746 4960 } else if (INTEL_INFO(dev)->gen >= 6) {
10d8d366 4961 intel_suspend_gt_powersave(dev);
e494837a 4962
4fc688ce 4963 mutex_lock(&dev_priv->rps.hw_lock);
38807746
D		 4964 if (IS_CHERRYVIEW(dev))
4965 cherryview_disable_rps(dev);
4966 else if (IS_VALLEYVIEW(dev))
d20d4f0c
JB		 4967 valleyview_disable_rps(dev);
4968 else
4969 gen6_disable_rps(dev);
c0951f0c 4970 dev_priv->rps.enabled = false;
4fc688ce 4971 mutex_unlock(&dev_priv->rps.hw_lock);
930ebb46 4972 }
8090c6b9
DV		 4973}
4974
1a01ab3b
JB		 4975static void intel_gen6_powersave_work(struct work_struct *work)
4976{
4977 struct drm_i915_private *dev_priv =
4978 container_of(work, struct drm_i915_private,
4979 rps.delayed_resume_work.work);
4980 struct drm_device *dev = dev_priv->dev;
4981
4fc688ce 4982 mutex_lock(&dev_priv->rps.hw_lock);
0a073b84 4983
38807746
D		 4984 if (IS_CHERRYVIEW(dev)) {
4985 cherryview_enable_rps(dev);
4986 } else if (IS_VALLEYVIEW(dev)) {
0a073b84 4987 valleyview_enable_rps(dev);
6edee7f3
BW		 4988 } else if (IS_BROADWELL(dev)) {
4989 gen8_enable_rps(dev);
c2bc2fc5 4990 __gen6_update_ring_freq(dev);
0a073b84
JB		 4991 } else {
4992 gen6_enable_rps(dev);
c2bc2fc5 4993 __gen6_update_ring_freq(dev);
0a073b84 4994 }
c0951f0c 4995 dev_priv->rps.enabled = true;
4fc688ce 4996 mutex_unlock(&dev_priv->rps.hw_lock);
c6df39b5
ID		 4997
4998 intel_runtime_pm_put(dev_priv);
1a01ab3b
JB		 4999}
5000
8090c6b9
DV		 5001void intel_enable_gt_powersave(struct drm_device *dev)
5002{
1a01ab3b
JB		 5003 struct drm_i915_private *dev_priv = dev->dev_private;
5004
8090c6b9 5005 if (IS_IRONLAKE_M(dev)) {
dc1d0136 5006 mutex_lock(&dev->struct_mutex);
8090c6b9
DV		 5007 ironlake_enable_drps(dev);
5008 ironlake_enable_rc6(dev);
5009 intel_init_emon(dev);
dc1d0136 5010 mutex_unlock(&dev->struct_mutex);
38807746 5011 } else if (INTEL_INFO(dev)->gen >= 6) {
1a01ab3b
JB		 5012 /*
5013 * PCU communication is slow and this doesn't need to be
5014 * done at any specific time, so do this out of our fast path
5015 * to make resume and init faster.
c6df39b5
ID		 5016 *
5017 * We depend on the HW RC6 power context save/restore
5018 * mechanism when entering D3 through runtime PM suspend. So
5019 * disable RPM until RPS/RC6 is properly setup. We can only
5020 * get here via the driver load/system resume/runtime resume
5021 * paths, so the _noresume version is enough (and in case of
5022 * runtime resume it's necessary).
1a01ab3b 5023 */
c6df39b5
ID		 5024 if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
5025 round_jiffies_up_relative(HZ)))
5026 intel_runtime_pm_get_noresume(dev_priv);
8090c6b9
DV		 5027 }
5028}
5029
c6df39b5
ID		 5030void intel_reset_gt_powersave(struct drm_device *dev)
5031{
5032 struct drm_i915_private *dev_priv = dev->dev_private;
5033
5034 dev_priv->rps.enabled = false;
5035 intel_enable_gt_powersave(dev);
5036}
5037
3107bd48
DV		 5038static void ibx_init_clock_gating(struct drm_device *dev)
5039{
5040 struct drm_i915_private *dev_priv = dev->dev_private;
5041
5042 /*
5043 * On Ibex Peak and Cougar Point, we need to disable clock
5044 * gating for the panel power sequencer or it will fail to
5045 * start up when no ports are active.
5046 */
5047 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
5048}
5049
0e088b8f
VS		 5050static void g4x_disable_trickle_feed(struct drm_device *dev)
5051{
5052 struct drm_i915_private *dev_priv = dev->dev_private;
5053 int pipe;
5054
5055 for_each_pipe(pipe) {
5056 I915_WRITE(DSPCNTR(pipe),
5057 I915_READ(DSPCNTR(pipe)) |
5058 DISPPLANE_TRICKLE_FEED_DISABLE);
1dba99f4 5059 intel_flush_primary_plane(dev_priv, pipe);
0e088b8f
VS		 5060 }
5061}
5062
017636cc
VS		 5063static void ilk_init_lp_watermarks(struct drm_device *dev)
5064{
5065 struct drm_i915_private *dev_priv = dev->dev_private;
5066
5067 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
5068 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
5069 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
5070
5071 /*
5072 * Don't touch WM1S_LP_EN here.
5073 * Doing so could cause underruns.
5074 */
5075}
5076
1fa61106 5077static void ironlake_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5078{
5079 struct drm_i915_private *dev_priv = dev->dev_private;
231e54f6 5080 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6f1d69b0 5081
f1e8fa56
DL		 5082 /*
5083 * Required for FBC
5084 * WaFbcDisableDpfcClockGating:ilk
5085 */
4d47e4f5
DL		 5086 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
5087 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
5088 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6f1d69b0
ED		 5089
5090 I915_WRITE(PCH_3DCGDIS0,
5091 MARIUNIT_CLOCK_GATE_DISABLE |
5092 SVSMUNIT_CLOCK_GATE_DISABLE);
5093 I915_WRITE(PCH_3DCGDIS1,
5094 VFMUNIT_CLOCK_GATE_DISABLE);
5095
6f1d69b0
ED		 5096 /*
5097 * According to the spec the following bits should be set in
5098 * order to enable memory self-refresh
5099 * The bit 22/21 of 0x42004
5100 * The bit 5 of 0x42020
5101 * The bit 15 of 0x45000
5102 */
5103 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5104 (I915_READ(ILK_DISPLAY_CHICKEN2) |
5105 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4d47e4f5 5106 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6f1d69b0
ED		 5107 I915_WRITE(DISP_ARB_CTL,
5108 (I915_READ(DISP_ARB_CTL) |
5109 DISP_FBC_WM_DIS));
017636cc
VS		 5110
5111 ilk_init_lp_watermarks(dev);
6f1d69b0
ED		 5112
5113 /*
5114 * Based on the document from hardware guys the following bits
5115 * should be set unconditionally in order to enable FBC.
5116 * The bit 22 of 0x42000
5117 * The bit 22 of 0x42004
5118 * The bit 7,8,9 of 0x42020.
5119 */
5120 if (IS_IRONLAKE_M(dev)) {
4bb35334 5121 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6f1d69b0
ED		 5122 I915_WRITE(ILK_DISPLAY_CHICKEN1,
5123 I915_READ(ILK_DISPLAY_CHICKEN1) |
5124 ILK_FBCQ_DIS);
5125 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5126 I915_READ(ILK_DISPLAY_CHICKEN2) |
5127 ILK_DPARB_GATE);
6f1d69b0
ED		 5128 }
5129
4d47e4f5
DL		 5130 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
5131
6f1d69b0
ED		 5132 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5133 I915_READ(ILK_DISPLAY_CHICKEN2) |
5134 ILK_ELPIN_409_SELECT);
5135 I915_WRITE(_3D_CHICKEN2,
5136 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
5137 _3D_CHICKEN2_WM_READ_PIPELINED);
4358a374 5138
ecdb4eb7 5139 /* WaDisableRenderCachePipelinedFlush:ilk */
4358a374
DV		 5140 I915_WRITE(CACHE_MODE_0,
5141 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
3107bd48 5142
4e04632e
AG		 5143 /* WaDisable_RenderCache_OperationalFlush:ilk */
5144 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5145
0e088b8f 5146 g4x_disable_trickle_feed(dev);
bdad2b2f 5147
3107bd48
DV		 5148 ibx_init_clock_gating(dev);
5149}
5150
5151static void cpt_init_clock_gating(struct drm_device *dev)
5152{
5153 struct drm_i915_private *dev_priv = dev->dev_private;
5154 int pipe;
3f704fa2 5155 uint32_t val;
3107bd48
DV		 5156
5157 /*
5158 * On Ibex Peak and Cougar Point, we need to disable clock
5159 * gating for the panel power sequencer or it will fail to
5160 * start up when no ports are active.
5161 */
cd664078
JB		 5162 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
5163 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
5164 PCH_CPUNIT_CLOCK_GATE_DISABLE);
3107bd48
DV		 5165 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
5166 DPLS_EDP_PPS_FIX_DIS);
335c07b7
TI		 5167 /* The below fixes the weird display corruption, a few pixels shifted
5168 * downward, on (only) LVDS of some HP laptops with IVY.
5169 */
3f704fa2 5170 for_each_pipe(pipe) {
dc4bd2d1
PZ		 5171 val = I915_READ(TRANS_CHICKEN2(pipe));
5172 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
5173 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
41aa3448 5174 if (dev_priv->vbt.fdi_rx_polarity_inverted)
3f704fa2 5175 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
dc4bd2d1
PZ		 5176 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
5177 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
5178 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
3f704fa2
PZ		 5179 I915_WRITE(TRANS_CHICKEN2(pipe), val);
5180 }
3107bd48
DV		 5181 /* WADP0ClockGatingDisable */
5182 for_each_pipe(pipe) {
5183 I915_WRITE(TRANS_CHICKEN1(pipe),
5184 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
5185 }
6f1d69b0
ED		 5186}
5187
1d7aaa0c
DV		 5188static void gen6_check_mch_setup(struct drm_device *dev)
5189{
5190 struct drm_i915_private *dev_priv = dev->dev_private;
5191 uint32_t tmp;
5192
5193 tmp = I915_READ(MCH_SSKPD);
5194 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) {
5195 DRM_INFO("Wrong MCH_SSKPD value: 0x%08x\n", tmp);
5196 DRM_INFO("This can cause pipe underruns and display issues.\n");
5197 DRM_INFO("Please upgrade your BIOS to fix this.\n");
5198 }
5199}
5200
1fa61106 5201static void gen6_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5202{
5203 struct drm_i915_private *dev_priv = dev->dev_private;
231e54f6 5204 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6f1d69b0 5205
231e54f6 5206 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6f1d69b0
ED		 5207
5208 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5209 I915_READ(ILK_DISPLAY_CHICKEN2) |
5210 ILK_ELPIN_409_SELECT);
5211
ecdb4eb7 5212 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
4283908e
DV		 5213 I915_WRITE(_3D_CHICKEN,
5214 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
5215
ecdb4eb7 5216 /* WaSetupGtModeTdRowDispatch:snb */
6547fbdb
DV		 5217 if (IS_SNB_GT1(dev))
5218 I915_WRITE(GEN6_GT_MODE,
5219 _MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
5220
4e04632e
AG		 5221 /* WaDisable_RenderCache_OperationalFlush:snb */
5222 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5223
8d85d272
VS		 5224 /*
5225 * BSpec recoomends 8x4 when MSAA is used,
5226 * however in practice 16x4 seems fastest.
c5c98a58
VS		 5227 *
5228 * Note that PS/WM thread counts depend on the WIZ hashing
5229 * disable bit, which we don't touch here, but it's good
5230 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
8d85d272
VS		 5231 */
5232 I915_WRITE(GEN6_GT_MODE,
5233 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
5234
017636cc 5235 ilk_init_lp_watermarks(dev);
6f1d69b0 5236
6f1d69b0 5237 I915_WRITE(CACHE_MODE_0,
50743298 5238 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
6f1d69b0
ED		 5239
5240 I915_WRITE(GEN6_UCGCTL1,
5241 I915_READ(GEN6_UCGCTL1) |
5242 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
5243 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
5244
5245 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
5246 * gating disable must be set. Failure to set it results in
5247 * flickering pixels due to Z write ordering failures after
5248 * some amount of runtime in the Mesa "fire" demo, and Unigine
5249 * Sanctuary and Tropics, and apparently anything else with
5250 * alpha test or pixel discard.
5251 *
5252 * According to the spec, bit 11 (RCCUNIT) must also be set,
5253 * but we didn't debug actual testcases to find it out.
0f846f81 5254 *
ef59318c
VS		 5255 * WaDisableRCCUnitClockGating:snb
5256 * WaDisableRCPBUnitClockGating:snb
6f1d69b0
ED		 5257 */
5258 I915_WRITE(GEN6_UCGCTL2,
5259 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
5260 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
5261
5eb146dd 5262 /* WaStripsFansDisableFastClipPerformanceFix:snb */
743b57d8
VS		 5263 I915_WRITE(_3D_CHICKEN3,
5264 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
6f1d69b0 5265
e927ecde
VS		 5266 /*
5267 * Bspec says:
5268 * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
5269 * 3DSTATE_SF number of SF output attributes is more than 16."
5270 */
5271 I915_WRITE(_3D_CHICKEN3,
5272 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
5273
6f1d69b0
ED		 5274 /*
5275 * According to the spec the following bits should be
5276 * set in order to enable memory self-refresh and fbc:
5277 * The bit21 and bit22 of 0x42000
5278 * The bit21 and bit22 of 0x42004
5279 * The bit5 and bit7 of 0x42020
5280 * The bit14 of 0x70180
5281 * The bit14 of 0x71180
4bb35334
DL		 5282 *
5283 * WaFbcAsynchFlipDisableFbcQueue:snb
6f1d69b0
ED		 5284 */
5285 I915_WRITE(ILK_DISPLAY_CHICKEN1,
5286 I915_READ(ILK_DISPLAY_CHICKEN1) |
5287 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
5288 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5289 I915_READ(ILK_DISPLAY_CHICKEN2) |
5290 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
231e54f6
DL		 5291 I915_WRITE(ILK_DSPCLK_GATE_D,
5292 I915_READ(ILK_DSPCLK_GATE_D) |
5293 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
5294 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
6f1d69b0 5295
0e088b8f 5296 g4x_disable_trickle_feed(dev);
f8f2ac9a 5297
3107bd48 5298 cpt_init_clock_gating(dev);
1d7aaa0c
DV		 5299
5300 gen6_check_mch_setup(dev);
6f1d69b0
ED		 5301}
5302
5303static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
5304{
5305 uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
5306
3aad9059 5307 /*
46680e0a 5308 * WaVSThreadDispatchOverride:ivb,vlv
3aad9059
VS		 5309 *
5310 * This actually overrides the dispatch
5311 * mode for all thread types.
5312 */
6f1d69b0
ED		 5313 reg &= ~GEN7_FF_SCHED_MASK;
5314 reg |= GEN7_FF_TS_SCHED_HW;
5315 reg |= GEN7_FF_VS_SCHED_HW;
5316 reg |= GEN7_FF_DS_SCHED_HW;
5317
5318 I915_WRITE(GEN7_FF_THREAD_MODE, reg);
5319}
5320
17a303ec
PZ		 5321static void lpt_init_clock_gating(struct drm_device *dev)
5322{
5323 struct drm_i915_private *dev_priv = dev->dev_private;
5324
5325 /*
5326 * TODO: this bit should only be enabled when really needed, then
5327 * disabled when not needed anymore in order to save power.
5328 */
5329 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
5330 I915_WRITE(SOUTH_DSPCLK_GATE_D,
5331 I915_READ(SOUTH_DSPCLK_GATE_D) |
5332 PCH_LP_PARTITION_LEVEL_DISABLE);
0a790cdb
PZ		 5333
5334 /* WADPOClockGatingDisable:hsw */
5335 I915_WRITE(_TRANSA_CHICKEN1,
5336 I915_READ(_TRANSA_CHICKEN1) |
5337 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
17a303ec
PZ		 5338}
5339
7d708ee4
ID		 5340static void lpt_suspend_hw(struct drm_device *dev)
5341{
5342 struct drm_i915_private *dev_priv = dev->dev_private;
5343
5344 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
5345 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
5346
5347 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
5348 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
5349 }
5350}
5351
1020a5c2
BW		 5352static void gen8_init_clock_gating(struct drm_device *dev)
5353{
5354 struct drm_i915_private *dev_priv = dev->dev_private;
07d27e20 5355 enum pipe pipe;
1020a5c2
BW		 5356
5357 I915_WRITE(WM3_LP_ILK, 0);
5358 I915_WRITE(WM2_LP_ILK, 0);
5359 I915_WRITE(WM1_LP_ILK, 0);
50ed5fbd
BW		 5360
5361 /* FIXME(BDW): Check all the w/a, some might only apply to
5362 * pre-production hw. */
5363
c8966e10
KG		 5364 /* WaDisablePartialInstShootdown:bdw */
5365 I915_WRITE(GEN8_ROW_CHICKEN,
5366 _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE));
5367
1411e6a5
KG		 5368 /* WaDisableThreadStallDopClockGating:bdw */
5369 /* FIXME: Unclear whether we really need this on production bdw. */
5370 I915_WRITE(GEN8_ROW_CHICKEN,
5371 _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
5372
4167e32c
DL		 5373 /*
5374 * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for
5375 * pre-production hardware
5376 */
fd392b60
BW		 5377 I915_WRITE(HALF_SLICE_CHICKEN3,
5378 _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS));
bf66347c
BW		 5379 I915_WRITE(HALF_SLICE_CHICKEN3,
5380 _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
4afe8d33
BW		 5381 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_BWGTLB_DISABLE));
5382
7f88da0c 5383 I915_WRITE(_3D_CHICKEN3,
b3f9ad93 5384 _MASKED_BIT_ENABLE(_3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(2)));
7f88da0c 5385
a75f3628
BW		 5386 I915_WRITE(COMMON_SLICE_CHICKEN2,
5387 _MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE));
5388
4c2e7a5f
BW		 5389 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
5390 _MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE));
5391
242a4018
BW		 5392 /* WaDisableDopClockGating:bdw May not be needed for production */
5393 I915_WRITE(GEN7_ROW_CHICKEN2,
5394 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5395
ab57fff1 5396 /* WaSwitchSolVfFArbitrationPriority:bdw */
50ed5fbd 5397 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
fe4ab3ce 5398
ab57fff1 5399 /* WaPsrDPAMaskVBlankInSRD:bdw */
fe4ab3ce
BW		 5400 I915_WRITE(CHICKEN_PAR1_1,
5401 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
5402
ab57fff1 5403 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
07d27e20
DL		 5404 for_each_pipe(pipe) {
5405 I915_WRITE(CHICKEN_PIPESL_1(pipe),
c7c65622 5406 I915_READ(CHICKEN_PIPESL_1(pipe)) |
8f670bb1 5407 BDW_DPRS_MASK_VBLANK_SRD);
fe4ab3ce 5408 }
63801f21
BW		 5409
5410 /* Use Force Non-Coherent whenever executing a 3D context. This is a
5411 * workaround for for a possible hang in the unlikely event a TLB
5412 * invalidation occurs during a PSD flush.
5413 */
5414 I915_WRITE(HDC_CHICKEN0,
5415 I915_READ(HDC_CHICKEN0) |
5416 _MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT));
ab57fff1
BW		 5417
5418 /* WaVSRefCountFullforceMissDisable:bdw */
5419 /* WaDSRefCountFullforceMissDisable:bdw */
5420 I915_WRITE(GEN7_FF_THREAD_MODE,
5421 I915_READ(GEN7_FF_THREAD_MODE) &
5422 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
36075a4c
VS		 5423
5424 /*
5425 * BSpec recommends 8x4 when MSAA is used,
5426 * however in practice 16x4 seems fastest.
c5c98a58
VS		 5427 *
5428 * Note that PS/WM thread counts depend on the WIZ hashing
5429 * disable bit, which we don't touch here, but it's good
5430 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
36075a4c
VS		 5431 */
5432 I915_WRITE(GEN7_GT_MODE,
5433 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
295e8bb7
VS		 5434
5435 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
5436 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4f1ca9e9
VS		 5437
5438 /* WaDisableSDEUnitClockGating:bdw */
5439 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
5440 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
5d708680
DL		 5441
5442 /* Wa4x4STCOptimizationDisable:bdw */
5443 I915_WRITE(CACHE_MODE_1,
5444 _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE));
1020a5c2
BW		 5445}
5446
cad2a2d7
ED		 5447static void haswell_init_clock_gating(struct drm_device *dev)
5448{
5449 struct drm_i915_private *dev_priv = dev->dev_private;
cad2a2d7 5450
017636cc 5451 ilk_init_lp_watermarks(dev);
cad2a2d7 5452
f3fc4884
FJ		 5453 /* L3 caching of data atomics doesn't work -- disable it. */
5454 I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
5455 I915_WRITE(HSW_ROW_CHICKEN3,
5456 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
5457
ecdb4eb7 5458 /* This is required by WaCatErrorRejectionIssue:hsw */
cad2a2d7
ED		 5459 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
5460 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
5461 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
5462
e36ea7ff
VS		 5463 /* WaVSRefCountFullforceMissDisable:hsw */
5464 I915_WRITE(GEN7_FF_THREAD_MODE,
5465 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
cad2a2d7 5466
4e04632e
AG		 5467 /* WaDisable_RenderCache_OperationalFlush:hsw */
5468 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5469
fe27c606
CW		 5470 /* enable HiZ Raw Stall Optimization */
5471 I915_WRITE(CACHE_MODE_0_GEN7,
5472 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
5473
ecdb4eb7 5474 /* WaDisable4x2SubspanOptimization:hsw */
cad2a2d7
ED		 5475 I915_WRITE(CACHE_MODE_1,
5476 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
1544d9d5 5477
a12c4967
VS		 5478 /*
5479 * BSpec recommends 8x4 when MSAA is used,
5480 * however in practice 16x4 seems fastest.
c5c98a58
VS		 5481 *
5482 * Note that PS/WM thread counts depend on the WIZ hashing
5483 * disable bit, which we don't touch here, but it's good
5484 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
a12c4967
VS		 5485 */
5486 I915_WRITE(GEN7_GT_MODE,
5487 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
5488
ecdb4eb7 5489 /* WaSwitchSolVfFArbitrationPriority:hsw */
e3dff585
BW		 5490 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
5491
90a88643
PZ		 5492 /* WaRsPkgCStateDisplayPMReq:hsw */
5493 I915_WRITE(CHICKEN_PAR1_1,
5494 I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
1544d9d5 5495
17a303ec 5496 lpt_init_clock_gating(dev);
cad2a2d7
ED		 5497}
5498
1fa61106 5499static void ivybridge_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5500{
5501 struct drm_i915_private *dev_priv = dev->dev_private;
20848223 5502 uint32_t snpcr;
6f1d69b0 5503
017636cc 5504 ilk_init_lp_watermarks(dev);
6f1d69b0 5505
231e54f6 5506 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
6f1d69b0 5507
ecdb4eb7 5508 /* WaDisableEarlyCull:ivb */
87f8020e
JB		 5509 I915_WRITE(_3D_CHICKEN3,
5510 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
5511
ecdb4eb7 5512 /* WaDisableBackToBackFlipFix:ivb */
6f1d69b0
ED		 5513 I915_WRITE(IVB_CHICKEN3,
5514 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
5515 CHICKEN3_DGMG_DONE_FIX_DISABLE);
5516
ecdb4eb7 5517 /* WaDisablePSDDualDispatchEnable:ivb */
12f3382b
JB		 5518 if (IS_IVB_GT1(dev))
5519 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
5520 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
12f3382b 5521
4e04632e
AG		 5522 /* WaDisable_RenderCache_OperationalFlush:ivb */
5523 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5524
ecdb4eb7 5525 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
6f1d69b0
ED		 5526 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
5527 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
5528
ecdb4eb7 5529 /* WaApplyL3ControlAndL3ChickenMode:ivb */
6f1d69b0
ED		 5530 I915_WRITE(GEN7_L3CNTLREG1,
5531 GEN7_WA_FOR_GEN7_L3_CONTROL);
5532 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
8ab43976
JB		 5533 GEN7_WA_L3_CHICKEN_MODE);
5534 if (IS_IVB_GT1(dev))
5535 I915_WRITE(GEN7_ROW_CHICKEN2,
5536 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
412236c2
VS		 5537 else {
5538 /* must write both registers */
5539 I915_WRITE(GEN7_ROW_CHICKEN2,
5540 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8ab43976
JB		 5541 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
5542 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
412236c2 5543 }
6f1d69b0 5544
ecdb4eb7 5545 /* WaForceL3Serialization:ivb */
61939d97
JB		 5546 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
5547 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
5548
1b80a19a 5549 /*
0f846f81 5550 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
ecdb4eb7 5551 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
0f846f81
JB		 5552 */
5553 I915_WRITE(GEN6_UCGCTL2,
28acf3b2 5554 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
0f846f81 5555
ecdb4eb7 5556 /* This is required by WaCatErrorRejectionIssue:ivb */
6f1d69b0
ED		 5557 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
5558 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
5559 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
5560
0e088b8f 5561 g4x_disable_trickle_feed(dev);
6f1d69b0
ED		 5562
5563 gen7_setup_fixed_func_scheduler(dev_priv);
97e1930f 5564
22721343
CW		 5565 if (0) { /* causes HiZ corruption on ivb:gt1 */
5566 /* enable HiZ Raw Stall Optimization */
5567 I915_WRITE(CACHE_MODE_0_GEN7,
5568 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
5569 }
116f2b6d 5570
ecdb4eb7 5571 /* WaDisable4x2SubspanOptimization:ivb */
97e1930f
DV		 5572 I915_WRITE(CACHE_MODE_1,
5573 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
20848223 5574
a607c1a4
VS		 5575 /*
5576 * BSpec recommends 8x4 when MSAA is used,
5577 * however in practice 16x4 seems fastest.
c5c98a58
VS		 5578 *
5579 * Note that PS/WM thread counts depend on the WIZ hashing
5580 * disable bit, which we don't touch here, but it's good
5581 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
a607c1a4
VS		 5582 */
5583 I915_WRITE(GEN7_GT_MODE,
5584 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
5585
20848223
BW		 5586 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
5587 snpcr &= ~GEN6_MBC_SNPCR_MASK;
5588 snpcr |= GEN6_MBC_SNPCR_MED;
5589 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3107bd48 5590
ab5c608b
BW		 5591 if (!HAS_PCH_NOP(dev))
5592 cpt_init_clock_gating(dev);
1d7aaa0c
DV		 5593
5594 gen6_check_mch_setup(dev);
6f1d69b0
ED		 5595}
5596
1fa61106 5597static void valleyview_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5598{
5599 struct drm_i915_private *dev_priv = dev->dev_private;
85b1d7b3
JB		 5600 u32 val;
5601
5602 mutex_lock(&dev_priv->rps.hw_lock);
5603 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
5604 mutex_unlock(&dev_priv->rps.hw_lock);
5605 switch ((val >> 6) & 3) {
5606 case 0:
f64a28a7 5607 case 1:
f6d51948 5608 dev_priv->mem_freq = 800;
85b1d7b3 5609 break;
f64a28a7 5610 case 2:
f6d51948 5611 dev_priv->mem_freq = 1066;
85b1d7b3 5612 break;
f64a28a7 5613 case 3:
2325991e 5614 dev_priv->mem_freq = 1333;
f64a28a7 5615 break;
85b1d7b3
JB		 5616 }
5617 DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
6f1d69b0 5618
d7fe0cc0 5619 I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
6f1d69b0 5620
ecdb4eb7 5621 /* WaDisableEarlyCull:vlv */
87f8020e
JB		 5622 I915_WRITE(_3D_CHICKEN3,
5623 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
5624
ecdb4eb7 5625 /* WaDisableBackToBackFlipFix:vlv */
6f1d69b0
ED		 5626 I915_WRITE(IVB_CHICKEN3,
5627 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
5628 CHICKEN3_DGMG_DONE_FIX_DISABLE);
5629
fad7d36e 5630 /* WaPsdDispatchEnable:vlv */
ecdb4eb7 5631 /* WaDisablePSDDualDispatchEnable:vlv */
12f3382b 5632 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
d3bc0303
JB		 5633 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
5634 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
12f3382b 5635
4e04632e
AG		 5636 /* WaDisable_RenderCache_OperationalFlush:vlv */
5637 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5638
ecdb4eb7 5639 /* WaForceL3Serialization:vlv */
61939d97
JB		 5640 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
5641 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
5642
ecdb4eb7 5643 /* WaDisableDopClockGating:vlv */
8ab43976
JB		 5644 I915_WRITE(GEN7_ROW_CHICKEN2,
5645 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5646
ecdb4eb7 5647 /* This is required by WaCatErrorRejectionIssue:vlv */
6f1d69b0
ED		 5648 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
5649 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
5650 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
5651
46680e0a
VS		 5652 gen7_setup_fixed_func_scheduler(dev_priv);
5653
3c0edaeb 5654 /*
0f846f81 5655 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
ecdb4eb7 5656 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
0f846f81
JB		 5657 */
5658 I915_WRITE(GEN6_UCGCTL2,
3c0edaeb 5659 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
0f846f81 5660
c98f5062
AG		 5661 /* WaDisableL3Bank2xClockGate:vlv
5662 * Disabling L3 clock gating- MMIO 940c[25] = 1
5663 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
5664 I915_WRITE(GEN7_UCGCTL4,
5665 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
e3f33d46 5666
e0d8d59b 5667 I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
6f1d69b0 5668
afd58e79
VS		 5669 /*
5670 * BSpec says this must be set, even though
5671 * WaDisable4x2SubspanOptimization isn't listed for VLV.
5672 */
6b26c86d
DV		 5673 I915_WRITE(CACHE_MODE_1,
5674 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
7983117f 5675
031994ee
VS		 5676 /*
5677 * WaIncreaseL3CreditsForVLVB0:vlv
5678 * This is the hardware default actually.
5679 */
5680 I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
5681
2d809570 5682 /*
ecdb4eb7 5683 * WaDisableVLVClockGating_VBIIssue:vlv
2d809570
JB		 5684 * Disable clock gating on th GCFG unit to prevent a delay
5685 * in the reporting of vblank events.
5686 */
7a0d1eed 5687 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
6f1d69b0
ED		 5688}
5689
a4565da8
VS		 5690static void cherryview_init_clock_gating(struct drm_device *dev)
5691{
5692 struct drm_i915_private *dev_priv = dev->dev_private;
5693
5694 I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
5695
5696 I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
dd811e70
VS		 5697
5698 /* WaDisablePartialInstShootdown:chv */
5699 I915_WRITE(GEN8_ROW_CHICKEN,
5700 _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE));
a7068025
VS		 5701
5702 /* WaDisableThreadStallDopClockGating:chv */
5703 I915_WRITE(GEN8_ROW_CHICKEN,
5704 _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
232ce337
VS		 5705
5706 /* WaVSRefCountFullforceMissDisable:chv */
5707 /* WaDSRefCountFullforceMissDisable:chv */
5708 I915_WRITE(GEN7_FF_THREAD_MODE,
5709 I915_READ(GEN7_FF_THREAD_MODE) &
5710 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
acea6f95
VS		 5711
5712 /* WaDisableSemaphoreAndSyncFlipWait:chv */
5713 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
5714 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
0846697c
VS		 5715
5716 /* WaDisableCSUnitClockGating:chv */
5717 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
5718 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
c631780f
VS		 5719
5720 /* WaDisableSDEUnitClockGating:chv */
5721 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
5722 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
e0d34ce7
RB		 5723
5724 /* WaDisableSamplerPowerBypass:chv (pre-production hw) */
5725 I915_WRITE(HALF_SLICE_CHICKEN3,
5726 _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
e4443e45
VS		 5727
5728 /* WaDisableGunitClockGating:chv (pre-production hw) */
5729 I915_WRITE(VLV_GUNIT_CLOCK_GATE, I915_READ(VLV_GUNIT_CLOCK_GATE) |
5730 GINT_DIS);
5731
5732 /* WaDisableFfDopClockGating:chv (pre-production hw) */
5733 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
5734 _MASKED_BIT_ENABLE(GEN8_FF_DOP_CLOCK_GATE_DISABLE));
5735
5736 /* WaDisableDopClockGating:chv (pre-production hw) */
5737 I915_WRITE(GEN7_ROW_CHICKEN2,
5738 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5739 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
5740 GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
a4565da8
VS		 5741}
5742
1fa61106 5743static void g4x_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5744{
5745 struct drm_i915_private *dev_priv = dev->dev_private;
5746 uint32_t dspclk_gate;
5747
5748 I915_WRITE(RENCLK_GATE_D1, 0);
5749 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
5750 GS_UNIT_CLOCK_GATE_DISABLE |
5751 CL_UNIT_CLOCK_GATE_DISABLE);
5752 I915_WRITE(RAMCLK_GATE_D, 0);
5753 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
5754 OVRUNIT_CLOCK_GATE_DISABLE |
5755 OVCUNIT_CLOCK_GATE_DISABLE;
5756 if (IS_GM45(dev))
5757 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
5758 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
4358a374
DV		 5759
5760 /* WaDisableRenderCachePipelinedFlush */
5761 I915_WRITE(CACHE_MODE_0,
5762 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
de1aa629 5763
4e04632e
AG		 5764 /* WaDisable_RenderCache_OperationalFlush:g4x */
5765 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5766
0e088b8f 5767 g4x_disable_trickle_feed(dev);
6f1d69b0
ED		 5768}
5769
1fa61106 5770static void crestline_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5771{
5772 struct drm_i915_private *dev_priv = dev->dev_private;
5773
5774 I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
5775 I915_WRITE(RENCLK_GATE_D2, 0);
5776 I915_WRITE(DSPCLK_GATE_D, 0);
5777 I915_WRITE(RAMCLK_GATE_D, 0);
5778 I915_WRITE16(DEUC, 0);
20f94967
VS		 5779 I915_WRITE(MI_ARB_STATE,
5780 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4e04632e
AG		 5781
5782 /* WaDisable_RenderCache_OperationalFlush:gen4 */
5783 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6f1d69b0
ED		 5784}
5785
1fa61106 5786static void broadwater_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5787{
5788 struct drm_i915_private *dev_priv = dev->dev_private;
5789
5790 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
5791 I965_RCC_CLOCK_GATE_DISABLE |
5792 I965_RCPB_CLOCK_GATE_DISABLE |
5793 I965_ISC_CLOCK_GATE_DISABLE |
5794 I965_FBC_CLOCK_GATE_DISABLE);
5795 I915_WRITE(RENCLK_GATE_D2, 0);
20f94967
VS		 5796 I915_WRITE(MI_ARB_STATE,
5797 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4e04632e
AG		 5798
5799 /* WaDisable_RenderCache_OperationalFlush:gen4 */
5800 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6f1d69b0
ED		 5801}
5802
1fa61106 5803static void gen3_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5804{
5805 struct drm_i915_private *dev_priv = dev->dev_private;
5806 u32 dstate = I915_READ(D_STATE);
5807
5808 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
5809 DSTATE_DOT_CLOCK_GATING;
5810 I915_WRITE(D_STATE, dstate);
13a86b85
CW		 5811
5812 if (IS_PINEVIEW(dev))
5813 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
974a3b0f
DV		 5814
5815 /* IIR "flip pending" means done if this bit is set */
5816 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
12fabbcb
VS		 5817
5818 /* interrupts should cause a wake up from C3 */
3299254f 5819 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
dbb42748
VS		 5820
5821 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
5822 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
6f1d69b0
ED		 5823}
5824
1fa61106 5825static void i85x_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5826{
5827 struct drm_i915_private *dev_priv = dev->dev_private;
5828
5829 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
54e472ae
VS		 5830
5831 /* interrupts should cause a wake up from C3 */
5832 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
5833 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
6f1d69b0
ED		 5834}
5835
1fa61106 5836static void i830_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5837{
5838 struct drm_i915_private *dev_priv = dev->dev_private;
5839
5840 I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
5841}
5842
6f1d69b0
ED		 5843void intel_init_clock_gating(struct drm_device *dev)
5844{
5845 struct drm_i915_private *dev_priv = dev->dev_private;
5846
5847 dev_priv->display.init_clock_gating(dev);
6f1d69b0
ED		 5848}
5849
7d708ee4
ID		 5850void intel_suspend_hw(struct drm_device *dev)
5851{
5852 if (HAS_PCH_LPT(dev))
5853 lpt_suspend_hw(dev);
5854}
5855
c1ca727f
ID		 5856#define for_each_power_well(i, power_well, domain_mask, power_domains) \
5857 for (i = 0; \
5858 i < (power_domains)->power_well_count && \
5859 ((power_well) = &(power_domains)->power_wells[i]); \
5860 i++) \
5861 if ((power_well)->domains & (domain_mask))
5862
5863#define for_each_power_well_rev(i, power_well, domain_mask, power_domains) \
5864 for (i = (power_domains)->power_well_count - 1; \
5865 i >= 0 && ((power_well) = &(power_domains)->power_wells[i]);\
5866 i--) \
5867 if ((power_well)->domains & (domain_mask))
5868
15d199ea
PZ		 5869/**
5870 * We should only use the power well if we explicitly asked the hardware to
5871 * enable it, so check if it's enabled and also check if we've requested it to
5872 * be enabled.
5873 */
da7e29bd 5874static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
c1ca727f
ID		 5875 struct i915_power_well *power_well)
5876{
c1ca727f
ID		 5877 return I915_READ(HSW_PWR_WELL_DRIVER) ==
5878 (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED);
5879}
5880
bfafe93a
ID		 5881bool intel_display_power_enabled_unlocked(struct drm_i915_private *dev_priv,
5882 enum intel_display_power_domain domain)
ddf9c536 5883{
ddf9c536 5884 struct i915_power_domains *power_domains;
b8c000d9
ID		 5885 struct i915_power_well *power_well;
5886 bool is_enabled;
5887 int i;
5888
5889 if (dev_priv->pm.suspended)
5890 return false;
ddf9c536
ID		 5891
5892 power_domains = &dev_priv->power_domains;
bfafe93a 5893
b8c000d9 5894 is_enabled = true;
bfafe93a 5895
b8c000d9
ID		 5896 for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
5897 if (power_well->always_on)
5898 continue;
ddf9c536 5899
bfafe93a 5900 if (!power_well->hw_enabled) {
b8c000d9
ID		 5901 is_enabled = false;
5902 break;
5903 }
5904 }
bfafe93a 5905
b8c000d9 5906 return is_enabled;
ddf9c536
ID		 5907}
5908
da7e29bd 5909bool intel_display_power_enabled(struct drm_i915_private *dev_priv,
b97186f0 5910 enum intel_display_power_domain domain)
15d199ea 5911{
c1ca727f 5912 struct i915_power_domains *power_domains;
bfafe93a 5913 bool ret;
882244a3 5914
c1ca727f
ID		 5915 power_domains = &dev_priv->power_domains;
5916
c1ca727f 5917 mutex_lock(&power_domains->lock);
bfafe93a 5918 ret = intel_display_power_enabled_unlocked(dev_priv, domain);
c1ca727f
ID		 5919 mutex_unlock(&power_domains->lock);
5920
bfafe93a 5921 return ret;
15d199ea
PZ		 5922}
5923
93c73e8c
ID		 5924/*
5925 * Starting with Haswell, we have a "Power Down Well" that can be turned off
5926 * when not needed anymore. We have 4 registers that can request the power well
5927 * to be enabled, and it will only be disabled if none of the registers is
5928 * requesting it to be enabled.
5929 */
d5e8fdc8
PZ		 5930static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
5931{
5932 struct drm_device *dev = dev_priv->dev;
5933 unsigned long irqflags;
5934
f9dcb0df
PZ		 5935 /*
5936 * After we re-enable the power well, if we touch VGA register 0x3d5
5937 * we'll get unclaimed register interrupts. This stops after we write
5938 * anything to the VGA MSR register. The vgacon module uses this
5939 * register all the time, so if we unbind our driver and, as a
5940 * consequence, bind vgacon, we'll get stuck in an infinite loop at
5941 * console_unlock(). So make here we touch the VGA MSR register, making
5942 * sure vgacon can keep working normally without triggering interrupts
5943 * and error messages.
5944 */
5945 vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
5946 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
5947 vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
5948
d5e8fdc8
PZ		 5949 if (IS_BROADWELL(dev)) {
5950 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
5951 I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_B),
5952 dev_priv->de_irq_mask[PIPE_B]);
5953 I915_WRITE(GEN8_DE_PIPE_IER(PIPE_B),
5954 ~dev_priv->de_irq_mask[PIPE_B] |
5955 GEN8_PIPE_VBLANK);
5956 I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_C),
5957 dev_priv->de_irq_mask[PIPE_C]);
5958 I915_WRITE(GEN8_DE_PIPE_IER(PIPE_C),
5959 ~dev_priv->de_irq_mask[PIPE_C] |
5960 GEN8_PIPE_VBLANK);
5961 POSTING_READ(GEN8_DE_PIPE_IER(PIPE_C));
5962 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
5963 }
5964}
5965
da7e29bd 5966static void hsw_set_power_well(struct drm_i915_private *dev_priv,
c1ca727f 5967 struct i915_power_well *power_well, bool enable)
d0d3e513 5968{
fa42e23c
PZ		 5969 bool is_enabled, enable_requested;
5970 uint32_t tmp;
d0d3e513 5971
fa42e23c 5972 tmp = I915_READ(HSW_PWR_WELL_DRIVER);
6aedd1f5
PZ		 5973 is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
5974 enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
d0d3e513 5975
fa42e23c
PZ		 5976 if (enable) {
5977 if (!enable_requested)
6aedd1f5
PZ		 5978 I915_WRITE(HSW_PWR_WELL_DRIVER,
5979 HSW_PWR_WELL_ENABLE_REQUEST);
d0d3e513 5980
fa42e23c
PZ		 5981 if (!is_enabled) {
5982 DRM_DEBUG_KMS("Enabling power well\n");
5983 if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
6aedd1f5 5984 HSW_PWR_WELL_STATE_ENABLED), 20))
fa42e23c
PZ		 5985 DRM_ERROR("Timeout enabling power well\n");
5986 }
596cc11e 5987
d5e8fdc8 5988 hsw_power_well_post_enable(dev_priv);
fa42e23c
PZ		 5989 } else {
5990 if (enable_requested) {
5991 I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
9dbd8feb 5992 POSTING_READ(HSW_PWR_WELL_DRIVER);
fa42e23c 5993 DRM_DEBUG_KMS("Requesting to disable the power well\n");
d0d3e513
ED		 5994 }
5995 }
fa42e23c 5996}
d0d3e513 5997
c6cb582e
ID		 5998static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
5999 struct i915_power_well *power_well)
6000{
6001 hsw_set_power_well(dev_priv, power_well, power_well->count > 0);
6002
6003 /*
6004 * We're taking over the BIOS, so clear any requests made by it since
6005 * the driver is in charge now.
6006 */
6007 if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
6008 I915_WRITE(HSW_PWR_WELL_BIOS, 0);
6009}
6010
6011static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
6012 struct i915_power_well *power_well)
6013{
c6cb582e
ID		 6014 hsw_set_power_well(dev_priv, power_well, true);
6015}
6016
6017static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
6018 struct i915_power_well *power_well)
6019{
6020 hsw_set_power_well(dev_priv, power_well, false);
c6cb582e
ID		 6021}
6022
a45f4466
ID		 6023static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
6024 struct i915_power_well *power_well)
6025{
6026}
6027
6028static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
6029 struct i915_power_well *power_well)
6030{
6031 return true;
6032}
6033
d2011dc8
VS		 6034static void vlv_set_power_well(struct drm_i915_private *dev_priv,
6035 struct i915_power_well *power_well, bool enable)
77961eb9 6036{
d2011dc8 6037 enum punit_power_well power_well_id = power_well->data;
77961eb9
ID		 6038 u32 mask;
6039 u32 state;
6040 u32 ctrl;
6041
6042 mask = PUNIT_PWRGT_MASK(power_well_id);
6043 state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
6044 PUNIT_PWRGT_PWR_GATE(power_well_id);
6045
6046 mutex_lock(&dev_priv->rps.hw_lock);
6047
6048#define COND \
6049 ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
6050
6051 if (COND)
6052 goto out;
6053
6054 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
6055 ctrl &= ~mask;
6056 ctrl |= state;
6057 vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
6058
6059 if (wait_for(COND, 100))
6060 DRM_ERROR("timout setting power well state %08x (%08x)\n",
6061 state,
6062 vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
6063
6064#undef COND
6065
6066out:
6067 mutex_unlock(&dev_priv->rps.hw_lock);
6068}
6069
6070static void vlv_power_well_sync_hw(struct drm_i915_private *dev_priv,
6071 struct i915_power_well *power_well)
6072{
6073 vlv_set_power_well(dev_priv, power_well, power_well->count > 0);
6074}
6075
6076static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
6077 struct i915_power_well *power_well)
6078{
6079 vlv_set_power_well(dev_priv, power_well, true);
6080}
6081
6082static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
6083 struct i915_power_well *power_well)
6084{
6085 vlv_set_power_well(dev_priv, power_well, false);
6086}
6087
6088static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
6089 struct i915_power_well *power_well)
6090{
6091 int power_well_id = power_well->data;
6092 bool enabled = false;
6093 u32 mask;
6094 u32 state;
6095 u32 ctrl;
6096
6097 mask = PUNIT_PWRGT_MASK(power_well_id);
6098 ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
6099
6100 mutex_lock(&dev_priv->rps.hw_lock);
6101
6102 state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
6103 /*
6104 * We only ever set the power-on and power-gate states, anything
6105 * else is unexpected.
6106 */
6107 WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
6108 state != PUNIT_PWRGT_PWR_GATE(power_well_id));
6109 if (state == ctrl)
6110 enabled = true;
6111
6112 /*
6113 * A transient state at this point would mean some unexpected party
6114 * is poking at the power controls too.
6115 */
6116 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
6117 WARN_ON(ctrl != state);
6118
6119 mutex_unlock(&dev_priv->rps.hw_lock);
6120
6121 return enabled;
6122}
6123
6124static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
6125 struct i915_power_well *power_well)
6126{
6127 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
6128
6129 vlv_set_power_well(dev_priv, power_well, true);
6130
6131 spin_lock_irq(&dev_priv->irq_lock);
6132 valleyview_enable_display_irqs(dev_priv);
6133 spin_unlock_irq(&dev_priv->irq_lock);
6134
6135 /*
0d116a29
ID		 6136 * During driver initialization/resume we can avoid restoring the
6137 * part of the HW/SW state that will be inited anyway explicitly.
77961eb9 6138 */
0d116a29
ID		 6139 if (dev_priv->power_domains.initializing)
6140 return;
6141
6142 intel_hpd_init(dev_priv->dev);
77961eb9
ID		 6143
6144 i915_redisable_vga_power_on(dev_priv->dev);
6145}
6146
6147static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
6148 struct i915_power_well *power_well)
6149{
77961eb9
ID		 6150 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
6151
6152 spin_lock_irq(&dev_priv->irq_lock);
77961eb9
ID		 6153 valleyview_disable_display_irqs(dev_priv);
6154 spin_unlock_irq(&dev_priv->irq_lock);
6155
77961eb9
ID		 6156 vlv_set_power_well(dev_priv, power_well, false);
6157}
6158
aa519f23
VS		 6159static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
6160 struct i915_power_well *power_well)
6161{
6162 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
6163
6164 /*
6165 * Enable the CRI clock source so we can get at the
6166 * display and the reference clock for VGA
6167 * hotplug / manual detection.
6168 */
6169 I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) |
6170 DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
6171 udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
6172
6173 vlv_set_power_well(dev_priv, power_well, true);
6174
6175 /*
6176 * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
6177 * 6. De-assert cmn_reset/side_reset. Same as VLV X0.
6178 * a. GUnit 0x2110 bit[0] set to 1 (def 0)
6179 * b. The other bits such as sfr settings / modesel may all
6180 * be set to 0.
6181 *
6182 * This should only be done on init and resume from S3 with
6183 * both PLLs disabled, or we risk losing DPIO and PLL
6184 * synchronization.
6185 */
6186 I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
6187}
6188
6189static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
6190 struct i915_power_well *power_well)
6191{
6192 struct drm_device *dev = dev_priv->dev;
6193 enum pipe pipe;
6194
6195 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
6196
6197 for_each_pipe(pipe)
6198 assert_pll_disabled(dev_priv, pipe);
6199
6200 /* Assert common reset */
6201 I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
6202
6203 vlv_set_power_well(dev_priv, power_well, false);
6204}
6205
25eaa003
ID		 6206static void check_power_well_state(struct drm_i915_private *dev_priv,
6207 struct i915_power_well *power_well)
6208{
6209 bool enabled = power_well->ops->is_enabled(dev_priv, power_well);
6210
6211 if (power_well->always_on || !i915.disable_power_well) {
6212 if (!enabled)
6213 goto mismatch;
6214
6215 return;
6216 }
6217
6218 if (enabled != (power_well->count > 0))
6219 goto mismatch;
6220
6221 return;
6222
6223mismatch:
6224 WARN(1, "state mismatch for '%s' (always_on %d hw state %d use-count %d disable_power_well %d\n",
6225 power_well->name, power_well->always_on, enabled,
6226 power_well->count, i915.disable_power_well);
6227}
6228
da7e29bd 6229void intel_display_power_get(struct drm_i915_private *dev_priv,
6765625e
VS		 6230 enum intel_display_power_domain domain)
6231{
83c00f55 6232 struct i915_power_domains *power_domains;
c1ca727f
ID		 6233 struct i915_power_well *power_well;
6234 int i;
6765625e 6235
9e6ea71a
PZ		 6236 intel_runtime_pm_get(dev_priv);
6237
83c00f55
ID		 6238 power_domains = &dev_priv->power_domains;
6239
6240 mutex_lock(&power_domains->lock);
1da51581 6241
25eaa003
ID		 6242 for_each_power_well(i, power_well, BIT(domain), power_domains) {
6243 if (!power_well->count++) {
6244 DRM_DEBUG_KMS("enabling %s\n", power_well->name);
c6cb582e 6245 power_well->ops->enable(dev_priv, power_well);
bfafe93a 6246 power_well->hw_enabled = true;
25eaa003
ID		 6247 }
6248
6249 check_power_well_state(dev_priv, power_well);
6250 }
1da51581 6251
ddf9c536
ID		 6252 power_domains->domain_use_count[domain]++;
6253
83c00f55 6254 mutex_unlock(&power_domains->lock);
6765625e
VS		 6255}
6256
da7e29bd 6257void intel_display_power_put(struct drm_i915_private *dev_priv,
6765625e
VS		 6258 enum intel_display_power_domain domain)
6259{
83c00f55 6260 struct i915_power_domains *power_domains;
c1ca727f
ID		 6261 struct i915_power_well *power_well;
6262 int i;
6765625e 6263
83c00f55
ID		 6264 power_domains = &dev_priv->power_domains;
6265
6266 mutex_lock(&power_domains->lock);
1da51581 6267
1da51581
ID		 6268 WARN_ON(!power_domains->domain_use_count[domain]);
6269 power_domains->domain_use_count[domain]--;
ddf9c536 6270
70bf407c
ID		 6271 for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
6272 WARN_ON(!power_well->count);
6273
25eaa003
ID		 6274 if (!--power_well->count && i915.disable_power_well) {
6275 DRM_DEBUG_KMS("disabling %s\n", power_well->name);
bfafe93a 6276 power_well->hw_enabled = false;
c6cb582e 6277 power_well->ops->disable(dev_priv, power_well);
25eaa003
ID		 6278 }
6279
6280 check_power_well_state(dev_priv, power_well);
70bf407c 6281 }
1da51581 6282
83c00f55 6283 mutex_unlock(&power_domains->lock);
9e6ea71a
PZ		 6284
6285 intel_runtime_pm_put(dev_priv);
6765625e
VS		 6286}
6287
83c00f55 6288static struct i915_power_domains *hsw_pwr;
a38911a3
WX		 6289
6290/* Display audio driver power well request */
74b0c2d7 6291int i915_request_power_well(void)
a38911a3 6292{
b4ed4484
ID		 6293 struct drm_i915_private *dev_priv;
6294
74b0c2d7
TI		 6295 if (!hsw_pwr)
6296 return -ENODEV;
a38911a3 6297
b4ed4484
ID		 6298 dev_priv = container_of(hsw_pwr, struct drm_i915_private,
6299 power_domains);
da7e29bd 6300 intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO);
74b0c2d7 6301 return 0;
a38911a3
WX		 6302}
6303EXPORT_SYMBOL_GPL(i915_request_power_well);
6304
6305/* Display audio driver power well release */
74b0c2d7 6306int i915_release_power_well(void)
a38911a3 6307{
b4ed4484
ID		 6308 struct drm_i915_private *dev_priv;
6309
74b0c2d7
TI		 6310 if (!hsw_pwr)
6311 return -ENODEV;
a38911a3 6312
b4ed4484
ID		 6313 dev_priv = container_of(hsw_pwr, struct drm_i915_private,
6314 power_domains);
da7e29bd 6315 intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO);
74b0c2d7 6316 return 0;
a38911a3
WX		 6317}
6318EXPORT_SYMBOL_GPL(i915_release_power_well);
6319
c149dcb5
JN		 6320/*
6321 * Private interface for the audio driver to get CDCLK in kHz.
6322 *
6323 * Caller must request power well using i915_request_power_well() prior to
6324 * making the call.
6325 */
6326int i915_get_cdclk_freq(void)
6327{
6328 struct drm_i915_private *dev_priv;
6329
6330 if (!hsw_pwr)
6331 return -ENODEV;
6332
6333 dev_priv = container_of(hsw_pwr, struct drm_i915_private,
6334 power_domains);
6335
6336 return intel_ddi_get_cdclk_freq(dev_priv);
6337}
6338EXPORT_SYMBOL_GPL(i915_get_cdclk_freq);
6339
6340
efcad917
ID		 6341#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1)
6342
6343#define HSW_ALWAYS_ON_POWER_DOMAINS ( \
6344 BIT(POWER_DOMAIN_PIPE_A) | \
f5938f36 6345 BIT(POWER_DOMAIN_TRANSCODER_EDP) | \
319be8ae
ID		 6346 BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) | \
6347 BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) | \
6348 BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \
6349 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \
6350 BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \
6351 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \
6352 BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \
6353 BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \
6354 BIT(POWER_DOMAIN_PORT_CRT) | \
f5938f36 6355 BIT(POWER_DOMAIN_INIT))
efcad917
ID		 6356#define HSW_DISPLAY_POWER_DOMAINS ( \
6357 (POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) | \
6358 BIT(POWER_DOMAIN_INIT))
6359
6360#define BDW_ALWAYS_ON_POWER_DOMAINS ( \
6361 HSW_ALWAYS_ON_POWER_DOMAINS | \
6362 BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER))
6363#define BDW_DISPLAY_POWER_DOMAINS ( \
6364 (POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS) | \
6365 BIT(POWER_DOMAIN_INIT))
6366
77961eb9
ID		 6367#define VLV_ALWAYS_ON_POWER_DOMAINS BIT(POWER_DOMAIN_INIT)
6368#define VLV_DISPLAY_POWER_DOMAINS POWER_DOMAIN_MASK
6369
6370#define VLV_DPIO_CMN_BC_POWER_DOMAINS ( \
6371 BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \
6372 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \
6373 BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \
6374 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \
6375 BIT(POWER_DOMAIN_PORT_CRT) | \
6376 BIT(POWER_DOMAIN_INIT))
6377
6378#define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS ( \
6379 BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \
6380 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \
6381 BIT(POWER_DOMAIN_INIT))
6382
6383#define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS ( \
6384 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \
6385 BIT(POWER_DOMAIN_INIT))
6386
6387#define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS ( \
6388 BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \
6389 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \
6390 BIT(POWER_DOMAIN_INIT))
6391
6392#define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS ( \
6393 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \
6394 BIT(POWER_DOMAIN_INIT))
6395
a45f4466
ID		 6396static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
6397 .sync_hw = i9xx_always_on_power_well_noop,
6398 .enable = i9xx_always_on_power_well_noop,
6399 .disable = i9xx_always_on_power_well_noop,
6400 .is_enabled = i9xx_always_on_power_well_enabled,
6401};
c6cb582e 6402
1c2256df
ID		 6403static struct i915_power_well i9xx_always_on_power_well[] = {
6404 {
6405 .name = "always-on",
6406 .always_on = 1,
6407 .domains = POWER_DOMAIN_MASK,
c6cb582e 6408 .ops = &i9xx_always_on_power_well_ops,
1c2256df
ID		 6409 },
6410};
6411
c6cb582e
ID		 6412static const struct i915_power_well_ops hsw_power_well_ops = {
6413 .sync_hw = hsw_power_well_sync_hw,
6414 .enable = hsw_power_well_enable,
6415 .disable = hsw_power_well_disable,
6416 .is_enabled = hsw_power_well_enabled,
6417};
6418
c1ca727f 6419static struct i915_power_well hsw_power_wells[] = {
6f3ef5dd
ID		 6420 {
6421 .name = "always-on",
6422 .always_on = 1,
6423 .domains = HSW_ALWAYS_ON_POWER_DOMAINS,
c6cb582e 6424 .ops = &i9xx_always_on_power_well_ops,
6f3ef5dd 6425 },
c1ca727f
ID		 6426 {
6427 .name = "display",
efcad917 6428 .domains = HSW_DISPLAY_POWER_DOMAINS,
c6cb582e 6429 .ops = &hsw_power_well_ops,
c1ca727f
ID		 6430 },
6431};
6432
6433static struct i915_power_well bdw_power_wells[] = {
6f3ef5dd
ID		 6434 {
6435 .name = "always-on",
6436 .always_on = 1,
6437 .domains = BDW_ALWAYS_ON_POWER_DOMAINS,
c6cb582e 6438 .ops = &i9xx_always_on_power_well_ops,
6f3ef5dd 6439 },
c1ca727f
ID		 6440 {
6441 .name = "display",
efcad917 6442 .domains = BDW_DISPLAY_POWER_DOMAINS,
c6cb582e 6443 .ops = &hsw_power_well_ops,
c1ca727f
ID		 6444 },
6445};
6446
77961eb9
ID		 6447static const struct i915_power_well_ops vlv_display_power_well_ops = {
6448 .sync_hw = vlv_power_well_sync_hw,
6449 .enable = vlv_display_power_well_enable,
6450 .disable = vlv_display_power_well_disable,
6451 .is_enabled = vlv_power_well_enabled,
6452};
6453
aa519f23
VS		 6454static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
6455 .sync_hw = vlv_power_well_sync_hw,
6456 .enable = vlv_dpio_cmn_power_well_enable,
6457 .disable = vlv_dpio_cmn_power_well_disable,
6458 .is_enabled = vlv_power_well_enabled,
6459};
6460
77961eb9
ID		 6461static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
6462 .sync_hw = vlv_power_well_sync_hw,
6463 .enable = vlv_power_well_enable,
6464 .disable = vlv_power_well_disable,
6465 .is_enabled = vlv_power_well_enabled,
6466};
6467
6468static struct i915_power_well vlv_power_wells[] = {
6469 {
6470 .name = "always-on",
6471 .always_on = 1,
6472 .domains = VLV_ALWAYS_ON_POWER_DOMAINS,
6473 .ops = &i9xx_always_on_power_well_ops,
6474 },
6475 {
6476 .name = "display",
6477 .domains = VLV_DISPLAY_POWER_DOMAINS,
6478 .data = PUNIT_POWER_WELL_DISP2D,
6479 .ops = &vlv_display_power_well_ops,
6480 },
77961eb9
ID		 6481 {
6482 .name = "dpio-tx-b-01",
6483 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
6484 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
6485 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
6486 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
6487 .ops = &vlv_dpio_power_well_ops,
6488 .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
6489 },
6490 {
6491 .name = "dpio-tx-b-23",
6492 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
6493 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
6494 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
6495 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
6496 .ops = &vlv_dpio_power_well_ops,
6497 .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
6498 },
6499 {
6500 .name = "dpio-tx-c-01",
6501 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
6502 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
6503 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
6504 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
6505 .ops = &vlv_dpio_power_well_ops,
6506 .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
6507 },
6508 {
6509 .name = "dpio-tx-c-23",
6510 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
6511 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
6512 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
6513 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
6514 .ops = &vlv_dpio_power_well_ops,
6515 .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
6516 },
f099a3c6
JB		 6517 {
6518 .name = "dpio-common",
6519 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
6520 .data = PUNIT_POWER_WELL_DPIO_CMN_BC,
aa519f23 6521 .ops = &vlv_dpio_cmn_power_well_ops,
f099a3c6 6522 },
77961eb9
ID		 6523};
6524
d2011dc8
VS		 6525static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv,
6526 enum punit_power_well power_well_id)
6527{
6528 struct i915_power_domains *power_domains = &dev_priv->power_domains;
6529 struct i915_power_well *power_well;
6530 int i;
6531
6532 for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
6533 if (power_well->data == power_well_id)
6534 return power_well;
6535 }
6536
6537 return NULL;
6538}
6539
c1ca727f
ID		 6540#define set_power_wells(power_domains, __power_wells) ({ \
6541 (power_domains)->power_wells = (__power_wells); \
6542 (power_domains)->power_well_count = ARRAY_SIZE(__power_wells); \
6543})
6544
da7e29bd 6545int intel_power_domains_init(struct drm_i915_private *dev_priv)
a38911a3 6546{
83c00f55 6547 struct i915_power_domains *power_domains = &dev_priv->power_domains;
c1ca727f 6548
83c00f55 6549 mutex_init(&power_domains->lock);
a38911a3 6550
c1ca727f
ID		 6551 /*
6552 * The enabling order will be from lower to higher indexed wells,
6553 * the disabling order is reversed.
6554 */
da7e29bd 6555 if (IS_HASWELL(dev_priv->dev)) {
c1ca727f
ID		 6556 set_power_wells(power_domains, hsw_power_wells);
6557 hsw_pwr = power_domains;
da7e29bd 6558 } else if (IS_BROADWELL(dev_priv->dev)) {
c1ca727f
ID		 6559 set_power_wells(power_domains, bdw_power_wells);
6560 hsw_pwr = power_domains;
77961eb9
ID		 6561 } else if (IS_VALLEYVIEW(dev_priv->dev)) {
6562 set_power_wells(power_domains, vlv_power_wells);
c1ca727f 6563 } else {
1c2256df 6564 set_power_wells(power_domains, i9xx_always_on_power_well);
c1ca727f 6565 }
a38911a3
WX		 6566
6567 return 0;
6568}
6569
da7e29bd 6570void intel_power_domains_remove(struct drm_i915_private *dev_priv)
a38911a3
WX		 6571{
6572 hsw_pwr = NULL;
6573}
6574
da7e29bd 6575static void intel_power_domains_resume(struct drm_i915_private *dev_priv)
9cdb826c 6576{
83c00f55
ID		 6577 struct i915_power_domains *power_domains = &dev_priv->power_domains;
6578 struct i915_power_well *power_well;
c1ca727f 6579 int i;
9cdb826c 6580
83c00f55 6581 mutex_lock(&power_domains->lock);
bfafe93a 6582 for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
a45f4466 6583 power_well->ops->sync_hw(dev_priv, power_well);
bfafe93a
ID		 6584 power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
6585 power_well);
6586 }
83c00f55 6587 mutex_unlock(&power_domains->lock);
a38911a3
WX		 6588}
6589
d2011dc8
VS		 6590static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
6591{
6592 struct i915_power_well *cmn =
6593 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
6594 struct i915_power_well *disp2d =
6595 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);
6596
6597 /* nothing to do if common lane is already off */
6598 if (!cmn->ops->is_enabled(dev_priv, cmn))
6599 return;
6600
6601 /* If the display might be already active skip this */
6602 if (disp2d->ops->is_enabled(dev_priv, disp2d) &&
6603 I915_READ(DPIO_CTL) & DPIO_CMNRST)
6604 return;
6605
6606 DRM_DEBUG_KMS("toggling display PHY side reset\n");
6607
6608 /* cmnlane needs DPLL registers */
6609 disp2d->ops->enable(dev_priv, disp2d);
6610
6611 /*
6612 * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
6613 * Need to assert and de-assert PHY SB reset by gating the
6614 * common lane power, then un-gating it.
6615 * Simply ungating isn't enough to reset the PHY enough to get
6616 * ports and lanes running.
6617 */
6618 cmn->ops->disable(dev_priv, cmn);
6619}
6620
da7e29bd 6621void intel_power_domains_init_hw(struct drm_i915_private *dev_priv)
d0d3e513 6622{
d2011dc8 6623 struct drm_device *dev = dev_priv->dev;
0d116a29
ID		 6624 struct i915_power_domains *power_domains = &dev_priv->power_domains;
6625
6626 power_domains->initializing = true;
d2011dc8
VS		 6627
6628 if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
6629 mutex_lock(&power_domains->lock);
6630 vlv_cmnlane_wa(dev_priv);
6631 mutex_unlock(&power_domains->lock);
6632 }
6633
fa42e23c 6634 /* For now, we need the power well to be always enabled. */
da7e29bd
ID		 6635 intel_display_set_init_power(dev_priv, true);
6636 intel_power_domains_resume(dev_priv);
0d116a29 6637 power_domains->initializing = false;
d0d3e513
ED		 6638}
6639
c67a470b
PZ		 6640void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv)
6641{
d361ae26 6642 intel_runtime_pm_get(dev_priv);
c67a470b
PZ		 6643}
6644
6645void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv)
6646{
d361ae26 6647 intel_runtime_pm_put(dev_priv);
c67a470b
PZ		 6648}
6649
8a187455
PZ		 6650void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
6651{
6652 struct drm_device *dev = dev_priv->dev;
6653 struct device *device = &dev->pdev->dev;
6654
6655 if (!HAS_RUNTIME_PM(dev))
6656 return;
6657
6658 pm_runtime_get_sync(device);
6659 WARN(dev_priv->pm.suspended, "Device still suspended.\n");
6660}
6661
c6df39b5
ID		 6662void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
6663{
6664 struct drm_device *dev = dev_priv->dev;
6665 struct device *device = &dev->pdev->dev;
6666
6667 if (!HAS_RUNTIME_PM(dev))
6668 return;
6669
6670 WARN(dev_priv->pm.suspended, "Getting nosync-ref while suspended.\n");
6671 pm_runtime_get_noresume(device);
6672}
6673
8a187455
PZ		 6674void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
6675{
6676 struct drm_device *dev = dev_priv->dev;
6677 struct device *device = &dev->pdev->dev;
6678
6679 if (!HAS_RUNTIME_PM(dev))
6680 return;
6681
6682 pm_runtime_mark_last_busy(device);
6683 pm_runtime_put_autosuspend(device);
6684}
6685
6686void intel_init_runtime_pm(struct drm_i915_private *dev_priv)
6687{
6688 struct drm_device *dev = dev_priv->dev;
6689 struct device *device = &dev->pdev->dev;
6690
8a187455
PZ		 6691 if (!HAS_RUNTIME_PM(dev))
6692 return;
6693
6694 pm_runtime_set_active(device);
6695
aeab0b5a
ID		 6696 /*
6697 * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
6698 * requirement.
6699 */
6700 if (!intel_enable_rc6(dev)) {
6701 DRM_INFO("RC6 disabled, disabling runtime PM support\n");
6702 return;
6703 }
6704
8a187455
PZ		 6705 pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */
6706 pm_runtime_mark_last_busy(device);
6707 pm_runtime_use_autosuspend(device);
ba0239e0
PZ		 6708
6709 pm_runtime_put_autosuspend(device);
8a187455
PZ		 6710}
6711
6712void intel_fini_runtime_pm(struct drm_i915_private *dev_priv)
6713{
6714 struct drm_device *dev = dev_priv->dev;
6715 struct device *device = &dev->pdev->dev;
6716
6717 if (!HAS_RUNTIME_PM(dev))
6718 return;
6719
aeab0b5a
ID		 6720 if (!intel_enable_rc6(dev))
6721 return;
6722
8a187455
PZ		 6723 /* Make sure we're not suspended first. */
6724 pm_runtime_get_sync(device);
6725 pm_runtime_disable(device);
6726}
6727
1fa61106
ED		 6728/* Set up chip specific power management-related functions */
6729void intel_init_pm(struct drm_device *dev)
6730{
6731 struct drm_i915_private *dev_priv = dev->dev_private;
6732
3a77c4c4 6733 if (HAS_FBC(dev)) {
40045465 6734 if (INTEL_INFO(dev)->gen >= 7) {
1fa61106 6735 dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
40045465
VS		 6736 dev_priv->display.enable_fbc = gen7_enable_fbc;
6737 dev_priv->display.disable_fbc = ironlake_disable_fbc;
6738 } else if (INTEL_INFO(dev)->gen >= 5) {
6739 dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
6740 dev_priv->display.enable_fbc = ironlake_enable_fbc;
1fa61106
ED		 6741 dev_priv->display.disable_fbc = ironlake_disable_fbc;
6742 } else if (IS_GM45(dev)) {
6743 dev_priv->display.fbc_enabled = g4x_fbc_enabled;
6744 dev_priv->display.enable_fbc = g4x_enable_fbc;
6745 dev_priv->display.disable_fbc = g4x_disable_fbc;
40045465 6746 } else {
1fa61106
ED		 6747 dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
6748 dev_priv->display.enable_fbc = i8xx_enable_fbc;
6749 dev_priv->display.disable_fbc = i8xx_disable_fbc;
993495ae
VS		 6750
6751 /* This value was pulled out of someone's hat */
6752 I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT);
1fa61106 6753 }
1fa61106
ED		 6754 }
6755
c921aba8
DV		 6756 /* For cxsr */
6757 if (IS_PINEVIEW(dev))
6758 i915_pineview_get_mem_freq(dev);
6759 else if (IS_GEN5(dev))
6760 i915_ironlake_get_mem_freq(dev);
6761
1fa61106
ED		 6762 /* For FIFO watermark updates */
6763 if (HAS_PCH_SPLIT(dev)) {
fa50ad61 6764 ilk_setup_wm_latency(dev);
53615a5e 6765
bd602544
VS		 6766 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
6767 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
6768 (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
6769 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
6770 dev_priv->display.update_wm = ilk_update_wm;
6771 dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
6772 } else {
6773 DRM_DEBUG_KMS("Failed to read display plane latency. "
6774 "Disable CxSR\n");
6775 }
6776
6777 if (IS_GEN5(dev))
1fa61106 6778 dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
bd602544 6779 else if (IS_GEN6(dev))
1fa61106 6780 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
bd602544 6781 else if (IS_IVYBRIDGE(dev))
1fa61106 6782 dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
bd602544 6783 else if (IS_HASWELL(dev))
cad2a2d7 6784 dev_priv->display.init_clock_gating = haswell_init_clock_gating;
bd602544 6785 else if (INTEL_INFO(dev)->gen == 8)
1020a5c2 6786 dev_priv->display.init_clock_gating = gen8_init_clock_gating;
a4565da8
VS		 6787 } else if (IS_CHERRYVIEW(dev)) {
6788 dev_priv->display.update_wm = valleyview_update_wm;
6789 dev_priv->display.init_clock_gating =
6790 cherryview_init_clock_gating;
1fa61106
ED		 6791 } else if (IS_VALLEYVIEW(dev)) {
6792 dev_priv->display.update_wm = valleyview_update_wm;
6793 dev_priv->display.init_clock_gating =
6794 valleyview_init_clock_gating;
1fa61106
ED		 6795 } else if (IS_PINEVIEW(dev)) {
6796 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
6797 dev_priv->is_ddr3,
6798 dev_priv->fsb_freq,
6799 dev_priv->mem_freq)) {
6800 DRM_INFO("failed to find known CxSR latency "
6801 "(found ddr%s fsb freq %d, mem freq %d), "
6802 "disabling CxSR\n",
6803 (dev_priv->is_ddr3 == 1) ? "3" : "2",
6804 dev_priv->fsb_freq, dev_priv->mem_freq);
6805 /* Disable CxSR and never update its watermark again */
5209b1f4 6806 intel_set_memory_cxsr(dev_priv, false);
1fa61106
ED		 6807 dev_priv->display.update_wm = NULL;
6808 } else
6809 dev_priv->display.update_wm = pineview_update_wm;
6810 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
6811 } else if (IS_G4X(dev)) {
6812 dev_priv->display.update_wm = g4x_update_wm;
6813 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
6814 } else if (IS_GEN4(dev)) {
6815 dev_priv->display.update_wm = i965_update_wm;
6816 if (IS_CRESTLINE(dev))
6817 dev_priv->display.init_clock_gating = crestline_init_clock_gating;
6818 else if (IS_BROADWATER(dev))
6819 dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
6820 } else if (IS_GEN3(dev)) {
6821 dev_priv->display.update_wm = i9xx_update_wm;
6822 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
6823 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
feb56b93
DV		 6824 } else if (IS_GEN2(dev)) {
6825 if (INTEL_INFO(dev)->num_pipes == 1) {
6826 dev_priv->display.update_wm = i845_update_wm;
1fa61106 6827 dev_priv->display.get_fifo_size = i845_get_fifo_size;
feb56b93
DV		 6828 } else {
6829 dev_priv->display.update_wm = i9xx_update_wm;
1fa61106 6830 dev_priv->display.get_fifo_size = i830_get_fifo_size;
feb56b93
DV		 6831 }
6832
6833 if (IS_I85X(dev) || IS_I865G(dev))
6834 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
6835 else
6836 dev_priv->display.init_clock_gating = i830_init_clock_gating;
6837 } else {
6838 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
1fa61106
ED		 6839 }
6840}
6841
42c0526c
BW		 6842int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val)
6843{
4fc688ce 6844 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
42c0526c
BW		 6845
6846 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6847 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
6848 return -EAGAIN;
6849 }
6850
6851 I915_WRITE(GEN6_PCODE_DATA, *val);
6852 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6853
6854 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6855 500)) {
6856 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
6857 return -ETIMEDOUT;
6858 }
6859
6860 *val = I915_READ(GEN6_PCODE_DATA);
6861 I915_WRITE(GEN6_PCODE_DATA, 0);
6862
6863 return 0;
6864}
6865
6866int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
6867{
4fc688ce 6868 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
42c0526c
BW		 6869
6870 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6871 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
6872 return -EAGAIN;
6873 }
6874
6875 I915_WRITE(GEN6_PCODE_DATA, val);
6876 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6877
6878 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6879 500)) {
6880 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
6881 return -ETIMEDOUT;
6882 }
6883
6884 I915_WRITE(GEN6_PCODE_DATA, 0);
6885
6886 return 0;
6887}
a0e4e199 6888
2ec3815f 6889int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val)
855ba3be 6890{
07ab118b 6891 int div;
855ba3be 6892
07ab118b 6893 /* 4 x czclk */
2ec3815f 6894 switch (dev_priv->mem_freq) {
855ba3be 6895 case 800:
07ab118b 6896 div = 10;
855ba3be
JB		 6897 break;
6898 case 1066:
07ab118b 6899 div = 12;
855ba3be
JB		 6900 break;
6901 case 1333:
07ab118b 6902 div = 16;
855ba3be
JB		 6903 break;
6904 default:
6905 return -1;
6906 }
6907
2ec3815f 6908 return DIV_ROUND_CLOSEST(dev_priv->mem_freq * (val + 6 - 0xbd), 4 * div);
855ba3be
JB		 6909}
6910
2ec3815f 6911int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val)
855ba3be 6912{
07ab118b 6913 int mul;
855ba3be 6914
07ab118b 6915 /* 4 x czclk */
2ec3815f 6916 switch (dev_priv->mem_freq) {
855ba3be 6917 case 800:
07ab118b 6918 mul = 10;
855ba3be
JB		 6919 break;
6920 case 1066:
07ab118b 6921 mul = 12;
855ba3be
JB		 6922 break;
6923 case 1333:
07ab118b 6924 mul = 16;
855ba3be
JB		 6925 break;
6926 default:
6927 return -1;
6928 }
6929
2ec3815f 6930 return DIV_ROUND_CLOSEST(4 * mul * val, dev_priv->mem_freq) + 0xbd - 6;
855ba3be
JB		 6931}
6932
f742a552 6933void intel_pm_setup(struct drm_device *dev)
907b28c5
CW		 6934{
6935 struct drm_i915_private *dev_priv = dev->dev_private;
6936
f742a552
DV		 6937 mutex_init(&dev_priv->rps.hw_lock);
6938
907b28c5
CW		 6939 INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
6940 intel_gen6_powersave_work);
5d584b2e 6941
33688d95 6942 dev_priv->pm.suspended = false;
5d584b2e 6943 dev_priv->pm.irqs_disabled = false;
907b28c5 6944}
ubuntu-bionic-kernel mirror