]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/gpu/drm/i915/intel_pm.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Enable the maxfifo PM5 mode when appropriate on CHV
[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / i915 / intel_pm.c
CommitLineData
85208be0
ED		 1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
2b4e57bd 28#include <linux/cpufreq.h>
85208be0
ED		 29#include "i915_drv.h"
30#include "intel_drv.h"
eb48eb00
DV		 31#include "../../../platform/x86/intel_ips.h"
32#include <linux/module.h>
85208be0 33
dc39fff7
BW		 34/**
35 * RC6 is a special power stage which allows the GPU to enter an very
36 * low-voltage mode when idle, using down to 0V while at this stage. This
37 * stage is entered automatically when the GPU is idle when RC6 support is
38 * enabled, and as soon as new workload arises GPU wakes up automatically as well.
39 *
40 * There are different RC6 modes available in Intel GPU, which differentiate
41 * among each other with the latency required to enter and leave RC6 and
42 * voltage consumed by the GPU in different states.
43 *
44 * The combination of the following flags define which states GPU is allowed
45 * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
46 * RC6pp is deepest RC6. Their support by hardware varies according to the
47 * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
48 * which brings the most power savings; deeper states save more power, but
49 * require higher latency to switch to and wake up.
50 */
51#define INTEL_RC6_ENABLE (1<<0)
52#define INTEL_RC6p_ENABLE (1<<1)
53#define INTEL_RC6pp_ENABLE (1<<2)
54
da2078cd
DL		 55static void gen9_init_clock_gating(struct drm_device *dev)
56{
acd5c346
DL		 57 struct drm_i915_private *dev_priv = dev->dev_private;
58
77719d28
DL		 59 /* WaEnableLbsSlaRetryTimerDecrement:skl */
60 I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
61 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
62}
91e41d16 63
45db2194 64static void skl_init_clock_gating(struct drm_device *dev)
da2078cd 65{
acd5c346 66 struct drm_i915_private *dev_priv = dev->dev_private;
3ca5da43 67
77719d28
DL		 68 gen9_init_clock_gating(dev);
69
3dcd020a
HN		 70 if (INTEL_REVID(dev) == SKL_REVID_A0) {
71 /*
72 * WaDisableSDEUnitClockGating:skl
9253c2e5 73 * WaSetGAPSunitClckGateDisable:skl
3dcd020a
HN		 74 */
75 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
9253c2e5 76 GEN8_GAPSUNIT_CLOCK_GATE_DISABLE |
3dcd020a
HN		 77 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
78 }
8bc0ccf6 79
2caa3b26 80 if (INTEL_REVID(dev) <= SKL_REVID_D0) {
81e231af
DL		 81 /* WaDisableHDCInvalidation:skl */
82 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
83 BDW_DISABLE_HDC_INVALIDATION);
84
2caa3b26
DL		 85 /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
86 I915_WRITE(FF_SLICE_CS_CHICKEN2,
87 I915_READ(FF_SLICE_CS_CHICKEN2) |
88 GEN9_TSG_BARRIER_ACK_DISABLE);
89 }
81e231af 90
8bc0ccf6
DL		 91 if (INTEL_REVID(dev) <= SKL_REVID_E0)
92 /* WaDisableLSQCROPERFforOCL:skl */
93 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
94 GEN8_LQSC_RO_PERF_DIS);
da2078cd
DL		 95}
96
c921aba8
DV		 97static void i915_pineview_get_mem_freq(struct drm_device *dev)
98{
50227e1c 99 struct drm_i915_private *dev_priv = dev->dev_private;
c921aba8
DV		 100 u32 tmp;
101
102 tmp = I915_READ(CLKCFG);
103
104 switch (tmp & CLKCFG_FSB_MASK) {
105 case CLKCFG_FSB_533:
106 dev_priv->fsb_freq = 533; /* 133*4 */
107 break;
108 case CLKCFG_FSB_800:
109 dev_priv->fsb_freq = 800; /* 200*4 */
110 break;
111 case CLKCFG_FSB_667:
112 dev_priv->fsb_freq = 667; /* 167*4 */
113 break;
114 case CLKCFG_FSB_400:
115 dev_priv->fsb_freq = 400; /* 100*4 */
116 break;
117 }
118
119 switch (tmp & CLKCFG_MEM_MASK) {
120 case CLKCFG_MEM_533:
121 dev_priv->mem_freq = 533;
122 break;
123 case CLKCFG_MEM_667:
124 dev_priv->mem_freq = 667;
125 break;
126 case CLKCFG_MEM_800:
127 dev_priv->mem_freq = 800;
128 break;
129 }
130
131 /* detect pineview DDR3 setting */
132 tmp = I915_READ(CSHRDDR3CTL);
133 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
134}
135
136static void i915_ironlake_get_mem_freq(struct drm_device *dev)
137{
50227e1c 138 struct drm_i915_private *dev_priv = dev->dev_private;
c921aba8
DV		 139 u16 ddrpll, csipll;
140
141 ddrpll = I915_READ16(DDRMPLL1);
142 csipll = I915_READ16(CSIPLL0);
143
144 switch (ddrpll & 0xff) {
145 case 0xc:
146 dev_priv->mem_freq = 800;
147 break;
148 case 0x10:
149 dev_priv->mem_freq = 1066;
150 break;
151 case 0x14:
152 dev_priv->mem_freq = 1333;
153 break;
154 case 0x18:
155 dev_priv->mem_freq = 1600;
156 break;
157 default:
158 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
159 ddrpll & 0xff);
160 dev_priv->mem_freq = 0;
161 break;
162 }
163
20e4d407 164 dev_priv->ips.r_t = dev_priv->mem_freq;
c921aba8
DV		 165
166 switch (csipll & 0x3ff) {
167 case 0x00c:
168 dev_priv->fsb_freq = 3200;
169 break;
170 case 0x00e:
171 dev_priv->fsb_freq = 3733;
172 break;
173 case 0x010:
174 dev_priv->fsb_freq = 4266;
175 break;
176 case 0x012:
177 dev_priv->fsb_freq = 4800;
178 break;
179 case 0x014:
180 dev_priv->fsb_freq = 5333;
181 break;
182 case 0x016:
183 dev_priv->fsb_freq = 5866;
184 break;
185 case 0x018:
186 dev_priv->fsb_freq = 6400;
187 break;
188 default:
189 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
190 csipll & 0x3ff);
191 dev_priv->fsb_freq = 0;
192 break;
193 }
194
195 if (dev_priv->fsb_freq == 3200) {
20e4d407 196 dev_priv->ips.c_m = 0;
c921aba8 197 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
20e4d407 198 dev_priv->ips.c_m = 1;
c921aba8 199 } else {
20e4d407 200 dev_priv->ips.c_m = 2;
c921aba8
DV		 201 }
202}
203
b445e3b0
ED		 204static const struct cxsr_latency cxsr_latency_table[] = {
205 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
206 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
207 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
208 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
209 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
210
211 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
212 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
213 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
214 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
215 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
216
217 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
218 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
219 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
220 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
221 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
222
223 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
224 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
225 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
226 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
227 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
228
229 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
230 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
231 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
232 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
233 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
234
235 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
236 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
237 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
238 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
239 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
240};
241
63c62275 242static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
b445e3b0
ED		 243 int is_ddr3,
244 int fsb,
245 int mem)
246{
247 const struct cxsr_latency *latency;
248 int i;
249
250 if (fsb == 0 || mem == 0)
251 return NULL;
252
253 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
254 latency = &cxsr_latency_table[i];
255 if (is_desktop == latency->is_desktop &&
256 is_ddr3 == latency->is_ddr3 &&
257 fsb == latency->fsb_freq && mem == latency->mem_freq)
258 return latency;
259 }
260
261 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
262
263 return NULL;
264}
265
cfb41411
VS		 266static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
267{
268 u32 val;
269
270 mutex_lock(&dev_priv->rps.hw_lock);
271
272 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
273 if (enable)
274 val |= DSP_MAXFIFO_PM5_ENABLE;
275 else
276 val &= ~DSP_MAXFIFO_PM5_ENABLE;
277 vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
278
279 mutex_unlock(&dev_priv->rps.hw_lock);
280}
281
5209b1f4 282void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
b445e3b0 283{
5209b1f4
ID		 284 struct drm_device *dev = dev_priv->dev;
285 u32 val;
b445e3b0 286
5209b1f4
ID		 287 if (IS_VALLEYVIEW(dev)) {
288 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
cfb41411
VS		 289 if (IS_CHERRYVIEW(dev))
290 chv_set_memory_pm5(dev_priv, enable);
5209b1f4
ID		 291 } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) {
292 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
293 } else if (IS_PINEVIEW(dev)) {
294 val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
295 val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
296 I915_WRITE(DSPFW3, val);
297 } else if (IS_I945G(dev) || IS_I945GM(dev)) {
298 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
299 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
300 I915_WRITE(FW_BLC_SELF, val);
301 } else if (IS_I915GM(dev)) {
302 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
303 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
304 I915_WRITE(INSTPM, val);
305 } else {
306 return;
307 }
b445e3b0 308
5209b1f4
ID		 309 DRM_DEBUG_KMS("memory self-refresh is %s\n",
310 enable ? "enabled" : "disabled");
b445e3b0
ED		 311}
312
313/*
314 * Latency for FIFO fetches is dependent on several factors:
315 * - memory configuration (speed, channels)
316 * - chipset
317 * - current MCH state
318 * It can be fairly high in some situations, so here we assume a fairly
319 * pessimal value. It's a tradeoff between extra memory fetches (if we
320 * set this value too high, the FIFO will fetch frequently to stay full)
321 * and power consumption (set it too low to save power and we might see
322 * FIFO underruns and display "flicker").
323 *
324 * A value of 5us seems to be a good balance; safe for very low end
325 * platforms but not overly aggressive on lower latency configs.
326 */
5aef6003 327static const int pessimal_latency_ns = 5000;
b445e3b0 328
b5004720
VS		 329#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
330 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
331
332static int vlv_get_fifo_size(struct drm_device *dev,
333 enum pipe pipe, int plane)
334{
335 struct drm_i915_private *dev_priv = dev->dev_private;
336 int sprite0_start, sprite1_start, size;
337
338 switch (pipe) {
339 uint32_t dsparb, dsparb2, dsparb3;
340 case PIPE_A:
341 dsparb = I915_READ(DSPARB);
342 dsparb2 = I915_READ(DSPARB2);
343 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
344 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
345 break;
346 case PIPE_B:
347 dsparb = I915_READ(DSPARB);
348 dsparb2 = I915_READ(DSPARB2);
349 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
350 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
351 break;
352 case PIPE_C:
353 dsparb2 = I915_READ(DSPARB2);
354 dsparb3 = I915_READ(DSPARB3);
355 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
356 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
357 break;
358 default:
359 return 0;
360 }
361
362 switch (plane) {
363 case 0:
364 size = sprite0_start;
365 break;
366 case 1:
367 size = sprite1_start - sprite0_start;
368 break;
369 case 2:
370 size = 512 - 1 - sprite1_start;
371 break;
372 default:
373 return 0;
374 }
375
376 DRM_DEBUG_KMS("Pipe %c %s %c FIFO size: %d\n",
377 pipe_name(pipe), plane == 0 ? "primary" : "sprite",
378 plane == 0 ? plane_name(pipe) : sprite_name(pipe, plane - 1),
379 size);
380
381 return size;
382}
383
1fa61106 384static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
b445e3b0
ED		 385{
386 struct drm_i915_private *dev_priv = dev->dev_private;
387 uint32_t dsparb = I915_READ(DSPARB);
388 int size;
389
390 size = dsparb & 0x7f;
391 if (plane)
392 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
393
394 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
395 plane ? "B" : "A", size);
396
397 return size;
398}
399
feb56b93 400static int i830_get_fifo_size(struct drm_device *dev, int plane)
b445e3b0
ED		 401{
402 struct drm_i915_private *dev_priv = dev->dev_private;
403 uint32_t dsparb = I915_READ(DSPARB);
404 int size;
405
406 size = dsparb & 0x1ff;
407 if (plane)
408 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
409 size >>= 1; /* Convert to cachelines */
410
411 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
412 plane ? "B" : "A", size);
413
414 return size;
415}
416
1fa61106 417static int i845_get_fifo_size(struct drm_device *dev, int plane)
b445e3b0
ED		 418{
419 struct drm_i915_private *dev_priv = dev->dev_private;
420 uint32_t dsparb = I915_READ(DSPARB);
421 int size;
422
423 size = dsparb & 0x7f;
424 size >>= 2; /* Convert to cachelines */
425
426 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
427 plane ? "B" : "A",
428 size);
429
430 return size;
431}
432
b445e3b0
ED		 433/* Pineview has different values for various configs */
434static const struct intel_watermark_params pineview_display_wm = {
e0f0273e
VS		 435 .fifo_size = PINEVIEW_DISPLAY_FIFO,
436 .max_wm = PINEVIEW_MAX_WM,
437 .default_wm = PINEVIEW_DFT_WM,
438 .guard_size = PINEVIEW_GUARD_WM,
439 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
b445e3b0
ED		 440};
441static const struct intel_watermark_params pineview_display_hplloff_wm = {
e0f0273e
VS		 442 .fifo_size = PINEVIEW_DISPLAY_FIFO,
443 .max_wm = PINEVIEW_MAX_WM,
444 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
445 .guard_size = PINEVIEW_GUARD_WM,
446 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
b445e3b0
ED		 447};
448static const struct intel_watermark_params pineview_cursor_wm = {
e0f0273e
VS		 449 .fifo_size = PINEVIEW_CURSOR_FIFO,
450 .max_wm = PINEVIEW_CURSOR_MAX_WM,
451 .default_wm = PINEVIEW_CURSOR_DFT_WM,
452 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
453 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
b445e3b0
ED		 454};
455static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
e0f0273e
VS		 456 .fifo_size = PINEVIEW_CURSOR_FIFO,
457 .max_wm = PINEVIEW_CURSOR_MAX_WM,
458 .default_wm = PINEVIEW_CURSOR_DFT_WM,
459 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
460 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
b445e3b0
ED		 461};
462static const struct intel_watermark_params g4x_wm_info = {
e0f0273e
VS		 463 .fifo_size = G4X_FIFO_SIZE,
464 .max_wm = G4X_MAX_WM,
465 .default_wm = G4X_MAX_WM,
466 .guard_size = 2,
467 .cacheline_size = G4X_FIFO_LINE_SIZE,
b445e3b0
ED		 468};
469static const struct intel_watermark_params g4x_cursor_wm_info = {
e0f0273e
VS		 470 .fifo_size = I965_CURSOR_FIFO,
471 .max_wm = I965_CURSOR_MAX_WM,
472 .default_wm = I965_CURSOR_DFT_WM,
473 .guard_size = 2,
474 .cacheline_size = G4X_FIFO_LINE_SIZE,
b445e3b0
ED		 475};
476static const struct intel_watermark_params valleyview_wm_info = {
e0f0273e
VS		 477 .fifo_size = VALLEYVIEW_FIFO_SIZE,
478 .max_wm = VALLEYVIEW_MAX_WM,
479 .default_wm = VALLEYVIEW_MAX_WM,
480 .guard_size = 2,
481 .cacheline_size = G4X_FIFO_LINE_SIZE,
b445e3b0
ED		 482};
483static const struct intel_watermark_params valleyview_cursor_wm_info = {
e0f0273e
VS		 484 .fifo_size = I965_CURSOR_FIFO,
485 .max_wm = VALLEYVIEW_CURSOR_MAX_WM,
486 .default_wm = I965_CURSOR_DFT_WM,
487 .guard_size = 2,
488 .cacheline_size = G4X_FIFO_LINE_SIZE,
b445e3b0
ED		 489};
490static const struct intel_watermark_params i965_cursor_wm_info = {
e0f0273e
VS		 491 .fifo_size = I965_CURSOR_FIFO,
492 .max_wm = I965_CURSOR_MAX_WM,
493 .default_wm = I965_CURSOR_DFT_WM,
494 .guard_size = 2,
495 .cacheline_size = I915_FIFO_LINE_SIZE,
b445e3b0
ED		 496};
497static const struct intel_watermark_params i945_wm_info = {
e0f0273e
VS		 498 .fifo_size = I945_FIFO_SIZE,
499 .max_wm = I915_MAX_WM,
500 .default_wm = 1,
501 .guard_size = 2,
502 .cacheline_size = I915_FIFO_LINE_SIZE,
b445e3b0
ED		 503};
504static const struct intel_watermark_params i915_wm_info = {
e0f0273e
VS		 505 .fifo_size = I915_FIFO_SIZE,
506 .max_wm = I915_MAX_WM,
507 .default_wm = 1,
508 .guard_size = 2,
509 .cacheline_size = I915_FIFO_LINE_SIZE,
b445e3b0 510};
9d539105 511static const struct intel_watermark_params i830_a_wm_info = {
e0f0273e
VS		 512 .fifo_size = I855GM_FIFO_SIZE,
513 .max_wm = I915_MAX_WM,
514 .default_wm = 1,
515 .guard_size = 2,
516 .cacheline_size = I830_FIFO_LINE_SIZE,
b445e3b0 517};
9d539105
VS		 518static const struct intel_watermark_params i830_bc_wm_info = {
519 .fifo_size = I855GM_FIFO_SIZE,
520 .max_wm = I915_MAX_WM/2,
521 .default_wm = 1,
522 .guard_size = 2,
523 .cacheline_size = I830_FIFO_LINE_SIZE,
524};
feb56b93 525static const struct intel_watermark_params i845_wm_info = {
e0f0273e
VS		 526 .fifo_size = I830_FIFO_SIZE,
527 .max_wm = I915_MAX_WM,
528 .default_wm = 1,
529 .guard_size = 2,
530 .cacheline_size = I830_FIFO_LINE_SIZE,
b445e3b0
ED		 531};
532
b445e3b0
ED		 533/**
534 * intel_calculate_wm - calculate watermark level
535 * @clock_in_khz: pixel clock
536 * @wm: chip FIFO params
537 * @pixel_size: display pixel size
538 * @latency_ns: memory latency for the platform
539 *
540 * Calculate the watermark level (the level at which the display plane will
541 * start fetching from memory again). Each chip has a different display
542 * FIFO size and allocation, so the caller needs to figure that out and pass
543 * in the correct intel_watermark_params structure.
544 *
545 * As the pixel clock runs, the FIFO will be drained at a rate that depends
546 * on the pixel size. When it reaches the watermark level, it'll start
547 * fetching FIFO line sized based chunks from memory until the FIFO fills
548 * past the watermark point. If the FIFO drains completely, a FIFO underrun
549 * will occur, and a display engine hang could result.
550 */
551static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
552 const struct intel_watermark_params *wm,
553 int fifo_size,
554 int pixel_size,
555 unsigned long latency_ns)
556{
557 long entries_required, wm_size;
558
559 /*
560 * Note: we need to make sure we don't overflow for various clock &
561 * latency values.
562 * clocks go from a few thousand to several hundred thousand.
563 * latency is usually a few thousand
564 */
565 entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
566 1000;
567 entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
568
569 DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
570
571 wm_size = fifo_size - (entries_required + wm->guard_size);
572
573 DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
574
575 /* Don't promote wm_size to unsigned... */
576 if (wm_size > (long)wm->max_wm)
577 wm_size = wm->max_wm;
578 if (wm_size <= 0)
579 wm_size = wm->default_wm;
d6feb196
VS		 580
581 /*
582 * Bspec seems to indicate that the value shouldn't be lower than
583 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
584 * Lets go for 8 which is the burst size since certain platforms
585 * already use a hardcoded 8 (which is what the spec says should be
586 * done).
587 */
588 if (wm_size <= 8)
589 wm_size = 8;
590
b445e3b0
ED		 591 return wm_size;
592}
593
594static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
595{
596 struct drm_crtc *crtc, *enabled = NULL;
597
70e1e0ec 598 for_each_crtc(dev, crtc) {
3490ea5d 599 if (intel_crtc_active(crtc)) {
b445e3b0
ED		 600 if (enabled)
601 return NULL;
602 enabled = crtc;
603 }
604 }
605
606 return enabled;
607}
608
46ba614c 609static void pineview_update_wm(struct drm_crtc *unused_crtc)
b445e3b0 610{
46ba614c 611 struct drm_device *dev = unused_crtc->dev;
b445e3b0
ED		 612 struct drm_i915_private *dev_priv = dev->dev_private;
613 struct drm_crtc *crtc;
614 const struct cxsr_latency *latency;
615 u32 reg;
616 unsigned long wm;
617
618 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
619 dev_priv->fsb_freq, dev_priv->mem_freq);
620 if (!latency) {
621 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
5209b1f4 622 intel_set_memory_cxsr(dev_priv, false);
b445e3b0
ED		 623 return;
624 }
625
626 crtc = single_enabled_crtc(dev);
627 if (crtc) {
241bfc38 628 const struct drm_display_mode *adjusted_mode;
59bea882 629 int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
241bfc38
DL		 630 int clock;
631
6e3c9717 632 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
241bfc38 633 clock = adjusted_mode->crtc_clock;
b445e3b0
ED		 634
635 /* Display SR */
636 wm = intel_calculate_wm(clock, &pineview_display_wm,
637 pineview_display_wm.fifo_size,
638 pixel_size, latency->display_sr);
639 reg = I915_READ(DSPFW1);
640 reg &= ~DSPFW_SR_MASK;
641 reg |= wm << DSPFW_SR_SHIFT;
642 I915_WRITE(DSPFW1, reg);
643 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
644
645 /* cursor SR */
646 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
647 pineview_display_wm.fifo_size,
648 pixel_size, latency->cursor_sr);
649 reg = I915_READ(DSPFW3);
650 reg &= ~DSPFW_CURSOR_SR_MASK;
651 reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
652 I915_WRITE(DSPFW3, reg);
653
654 /* Display HPLL off SR */
655 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
656 pineview_display_hplloff_wm.fifo_size,
657 pixel_size, latency->display_hpll_disable);
658 reg = I915_READ(DSPFW3);
659 reg &= ~DSPFW_HPLL_SR_MASK;
660 reg |= wm & DSPFW_HPLL_SR_MASK;
661 I915_WRITE(DSPFW3, reg);
662
663 /* cursor HPLL off SR */
664 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
665 pineview_display_hplloff_wm.fifo_size,
666 pixel_size, latency->cursor_hpll_disable);
667 reg = I915_READ(DSPFW3);
668 reg &= ~DSPFW_HPLL_CURSOR_MASK;
669 reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
670 I915_WRITE(DSPFW3, reg);
671 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
672
5209b1f4 673 intel_set_memory_cxsr(dev_priv, true);
b445e3b0 674 } else {
5209b1f4 675 intel_set_memory_cxsr(dev_priv, false);
b445e3b0
ED		 676 }
677}
678
679static bool g4x_compute_wm0(struct drm_device *dev,
680 int plane,
681 const struct intel_watermark_params *display,
682 int display_latency_ns,
683 const struct intel_watermark_params *cursor,
684 int cursor_latency_ns,
685 int *plane_wm,
686 int *cursor_wm)
687{
688 struct drm_crtc *crtc;
4fe8590a 689 const struct drm_display_mode *adjusted_mode;
b445e3b0
ED		 690 int htotal, hdisplay, clock, pixel_size;
691 int line_time_us, line_count;
692 int entries, tlb_miss;
693
694 crtc = intel_get_crtc_for_plane(dev, plane);
3490ea5d 695 if (!intel_crtc_active(crtc)) {
b445e3b0
ED		 696 *cursor_wm = cursor->guard_size;
697 *plane_wm = display->guard_size;
698 return false;
699 }
700
6e3c9717 701 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
241bfc38 702 clock = adjusted_mode->crtc_clock;
fec8cba3 703 htotal = adjusted_mode->crtc_htotal;
6e3c9717 704 hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
59bea882 705 pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
b445e3b0
ED		 706
707 /* Use the small buffer method to calculate plane watermark */
708 entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
709 tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
710 if (tlb_miss > 0)
711 entries += tlb_miss;
712 entries = DIV_ROUND_UP(entries, display->cacheline_size);
713 *plane_wm = entries + display->guard_size;
714 if (*plane_wm > (int)display->max_wm)
715 *plane_wm = display->max_wm;
716
717 /* Use the large buffer method to calculate cursor watermark */
922044c9 718 line_time_us = max(htotal * 1000 / clock, 1);
b445e3b0 719 line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
3dd512fb 720 entries = line_count * crtc->cursor->state->crtc_w * pixel_size;
b445e3b0
ED		 721 tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
722 if (tlb_miss > 0)
723 entries += tlb_miss;
724 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
725 *cursor_wm = entries + cursor->guard_size;
726 if (*cursor_wm > (int)cursor->max_wm)
727 *cursor_wm = (int)cursor->max_wm;
728
729 return true;
730}
731
732/*
733 * Check the wm result.
734 *
735 * If any calculated watermark values is larger than the maximum value that
736 * can be programmed into the associated watermark register, that watermark
737 * must be disabled.
738 */
739static bool g4x_check_srwm(struct drm_device *dev,
740 int display_wm, int cursor_wm,
741 const struct intel_watermark_params *display,
742 const struct intel_watermark_params *cursor)
743{
744 DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
745 display_wm, cursor_wm);
746
747 if (display_wm > display->max_wm) {
748 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
749 display_wm, display->max_wm);
750 return false;
751 }
752
753 if (cursor_wm > cursor->max_wm) {
754 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
755 cursor_wm, cursor->max_wm);
756 return false;
757 }
758
759 if (!(display_wm || cursor_wm)) {
760 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
761 return false;
762 }
763
764 return true;
765}
766
767static bool g4x_compute_srwm(struct drm_device *dev,
768 int plane,
769 int latency_ns,
770 const struct intel_watermark_params *display,
771 const struct intel_watermark_params *cursor,
772 int *display_wm, int *cursor_wm)
773{
774 struct drm_crtc *crtc;
4fe8590a 775 const struct drm_display_mode *adjusted_mode;
b445e3b0
ED		 776 int hdisplay, htotal, pixel_size, clock;
777 unsigned long line_time_us;
778 int line_count, line_size;
779 int small, large;
780 int entries;
781
782 if (!latency_ns) {
783 *display_wm = *cursor_wm = 0;
784 return false;
785 }
786
787 crtc = intel_get_crtc_for_plane(dev, plane);
6e3c9717 788 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
241bfc38 789 clock = adjusted_mode->crtc_clock;
fec8cba3 790 htotal = adjusted_mode->crtc_htotal;
6e3c9717 791 hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
59bea882 792 pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
b445e3b0 793
922044c9 794 line_time_us = max(htotal * 1000 / clock, 1);
b445e3b0
ED		 795 line_count = (latency_ns / line_time_us + 1000) / 1000;
796 line_size = hdisplay * pixel_size;
797
798 /* Use the minimum of the small and large buffer method for primary */
799 small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
800 large = line_count * line_size;
801
802 entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
803 *display_wm = entries + display->guard_size;
804
805 /* calculate the self-refresh watermark for display cursor */
3dd512fb 806 entries = line_count * pixel_size * crtc->cursor->state->crtc_w;
b445e3b0
ED		 807 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
808 *cursor_wm = entries + cursor->guard_size;
809
810 return g4x_check_srwm(dev,
811 *display_wm, *cursor_wm,
812 display, cursor);
813}
814
0018fda1
VS		 815static void vlv_write_wm_values(struct intel_crtc *crtc,
816 const struct vlv_wm_values *wm)
817{
818 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
819 enum pipe pipe = crtc->pipe;
820
821 I915_WRITE(VLV_DDL(pipe),
822 (wm->ddl[pipe].cursor << DDL_CURSOR_SHIFT) |
823 (wm->ddl[pipe].sprite[1] << DDL_SPRITE_SHIFT(1)) |
824 (wm->ddl[pipe].sprite[0] << DDL_SPRITE_SHIFT(0)) |
825 (wm->ddl[pipe].primary << DDL_PLANE_SHIFT));
826
ae80152d
VS		 827 I915_WRITE(DSPFW1,
828 ((wm->sr.plane << DSPFW_SR_SHIFT) & DSPFW_SR_MASK) |
829 ((wm->pipe[PIPE_B].cursor << DSPFW_CURSORB_SHIFT) & DSPFW_CURSORB_MASK) |
830 ((wm->pipe[PIPE_B].primary << DSPFW_PLANEB_SHIFT) & DSPFW_PLANEB_MASK_VLV) |
831 ((wm->pipe[PIPE_A].primary << DSPFW_PLANEA_SHIFT) & DSPFW_PLANEA_MASK_VLV));
832 I915_WRITE(DSPFW2,
833 ((wm->pipe[PIPE_A].sprite[1] << DSPFW_SPRITEB_SHIFT) & DSPFW_SPRITEB_MASK_VLV) |
834 ((wm->pipe[PIPE_A].cursor << DSPFW_CURSORA_SHIFT) & DSPFW_CURSORA_MASK) |
835 ((wm->pipe[PIPE_A].sprite[0] << DSPFW_SPRITEA_SHIFT) & DSPFW_SPRITEA_MASK_VLV));
836 I915_WRITE(DSPFW3,
837 ((wm->sr.cursor << DSPFW_CURSOR_SR_SHIFT) & DSPFW_CURSOR_SR_MASK));
838
839 if (IS_CHERRYVIEW(dev_priv)) {
840 I915_WRITE(DSPFW7_CHV,
841 ((wm->pipe[PIPE_B].sprite[1] << DSPFW_SPRITED_SHIFT) & DSPFW_SPRITED_MASK) |
842 ((wm->pipe[PIPE_B].sprite[0] << DSPFW_SPRITEC_SHIFT) & DSPFW_SPRITEC_MASK));
843 I915_WRITE(DSPFW8_CHV,
844 ((wm->pipe[PIPE_C].sprite[1] << DSPFW_SPRITEF_SHIFT) & DSPFW_SPRITEF_MASK) |
845 ((wm->pipe[PIPE_C].sprite[0] << DSPFW_SPRITEE_SHIFT) & DSPFW_SPRITEE_MASK));
846 I915_WRITE(DSPFW9_CHV,
847 ((wm->pipe[PIPE_C].primary << DSPFW_PLANEC_SHIFT) & DSPFW_PLANEC_MASK) |
848 ((wm->pipe[PIPE_C].cursor << DSPFW_CURSORC_SHIFT) & DSPFW_CURSORC_MASK));
849 I915_WRITE(DSPHOWM,
850 (((wm->sr.plane >> 9) << DSPFW_SR_HI_SHIFT) & DSPFW_SR_HI_MASK) |
851 (((wm->pipe[PIPE_C].sprite[1] >> 8) << DSPFW_SPRITEF_HI_SHIFT) & DSPFW_SPRITEF_HI_MASK) |
852 (((wm->pipe[PIPE_C].sprite[0] >> 8) << DSPFW_SPRITEE_HI_SHIFT) & DSPFW_SPRITEE_HI_MASK) |
853 (((wm->pipe[PIPE_C].primary >> 8) << DSPFW_PLANEC_HI_SHIFT) & DSPFW_PLANEC_HI_MASK) |
854 (((wm->pipe[PIPE_B].sprite[1] >> 8) << DSPFW_SPRITED_HI_SHIFT) & DSPFW_SPRITED_HI_MASK) |
855 (((wm->pipe[PIPE_B].sprite[0] >> 8) << DSPFW_SPRITEC_HI_SHIFT) & DSPFW_SPRITEC_HI_MASK) |
856 (((wm->pipe[PIPE_B].primary >> 8) << DSPFW_PLANEB_HI_SHIFT) & DSPFW_PLANEB_HI_MASK) |
857 (((wm->pipe[PIPE_A].sprite[1] >> 8) << DSPFW_SPRITEB_HI_SHIFT) & DSPFW_SPRITEB_HI_MASK) |
858 (((wm->pipe[PIPE_A].sprite[0] >> 8) << DSPFW_SPRITEA_HI_SHIFT) & DSPFW_SPRITEA_HI_MASK) |
859 (((wm->pipe[PIPE_A].primary >> 8) << DSPFW_PLANEA_HI_SHIFT) & DSPFW_PLANEA_HI_MASK));
860 } else {
861 I915_WRITE(DSPFW7,
862 ((wm->pipe[PIPE_B].sprite[1] << DSPFW_SPRITED_SHIFT) & DSPFW_SPRITED_MASK) |
863 ((wm->pipe[PIPE_B].sprite[0] << DSPFW_SPRITEC_SHIFT) & DSPFW_SPRITEC_MASK));
864 I915_WRITE(DSPHOWM,
865 (((wm->sr.plane >> 9) << DSPFW_SR_HI_SHIFT) & DSPFW_SR_HI_MASK) |
866 (((wm->pipe[PIPE_B].sprite[1] >> 8) << DSPFW_SPRITED_HI_SHIFT) & DSPFW_SPRITED_HI_MASK) |
867 (((wm->pipe[PIPE_B].sprite[0] >> 8) << DSPFW_SPRITEC_HI_SHIFT) & DSPFW_SPRITEC_HI_MASK) |
868 (((wm->pipe[PIPE_B].primary >> 8) << DSPFW_PLANEB_HI_SHIFT) & DSPFW_PLANEB_HI_MASK) |
869 (((wm->pipe[PIPE_A].sprite[1] >> 8) << DSPFW_SPRITEB_HI_SHIFT) & DSPFW_SPRITEB_HI_MASK) |
870 (((wm->pipe[PIPE_A].sprite[0] >> 8) << DSPFW_SPRITEA_HI_SHIFT) & DSPFW_SPRITEA_HI_MASK) |
871 (((wm->pipe[PIPE_A].primary >> 8) << DSPFW_PLANEA_HI_SHIFT) & DSPFW_PLANEA_HI_MASK));
872 }
873
874 POSTING_READ(DSPFW1);
875
0018fda1
VS		 876 dev_priv->wm.vlv = *wm;
877}
878
341c526f 879static uint8_t vlv_compute_drain_latency(struct drm_crtc *crtc,
883a3d2f 880 struct drm_plane *plane)
b445e3b0 881{
5e56ba45 882 struct drm_device *dev = crtc->dev;
883a3d2f
VS		 883 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
884 int entries, prec_mult, drain_latency, pixel_size;
885 int clock = intel_crtc->config->base.adjusted_mode.crtc_clock;
341c526f 886 const int high_precision = IS_CHERRYVIEW(dev) ? 16 : 64;
b445e3b0 887
883a3d2f
VS		 888 /*
889 * FIXME the plane might have an fb
890 * but be invisible (eg. due to clipping)
891 */
892 if (!intel_crtc->active || !plane->state->fb)
893 return 0;
894
0948c265 895 if (WARN(clock == 0, "Pixel clock is zero!\n"))
341c526f 896 return 0;
b445e3b0 897
883a3d2f
VS		 898 pixel_size = drm_format_plane_cpp(plane->state->fb->pixel_format, 0);
899
0948c265 900 if (WARN(pixel_size == 0, "Pixel size is zero!\n"))
341c526f 901 return 0;
b445e3b0 902
a398e9c7 903 entries = DIV_ROUND_UP(clock, 1000) * pixel_size;
abfc00b5 904
341c526f
VS		 905 prec_mult = high_precision;
906 drain_latency = 64 * prec_mult * 4 / entries;
b445e3b0 907
341c526f
VS		 908 if (drain_latency > DRAIN_LATENCY_MASK) {
909 prec_mult /= 2;
910 drain_latency = 64 * prec_mult * 4 / entries;
abfc00b5
VS		 911 }
912
341c526f
VS		 913 if (drain_latency > DRAIN_LATENCY_MASK)
914 drain_latency = DRAIN_LATENCY_MASK;
b445e3b0 915
341c526f
VS		 916 return drain_latency | (prec_mult == high_precision ?
917 DDL_PRECISION_HIGH : DDL_PRECISION_LOW);
b445e3b0
ED		 918}
919
ae80152d
VS		 920static int vlv_compute_wm(struct intel_crtc *crtc,
921 struct intel_plane *plane,
922 int fifo_size)
b445e3b0 923{
ae80152d 924 int clock, entries, pixel_size;
b445e3b0 925
ae80152d
VS		 926 /*
927 * FIXME the plane might have an fb
928 * but be invisible (eg. due to clipping)
929 */
930 if (!crtc->active || !plane->base.state->fb)
931 return 0;
0948c265 932
ae80152d
VS		 933 pixel_size = drm_format_plane_cpp(plane->base.state->fb->pixel_format, 0);
934 clock = crtc->config->base.adjusted_mode.crtc_clock;
b445e3b0 935
ae80152d 936 entries = DIV_ROUND_UP(clock, 1000) * pixel_size;
b445e3b0 937
ae80152d
VS		 938 /*
939 * Set up the watermark such that we don't start issuing memory
940 * requests until we are within PND's max deadline value (256us).
941 * Idea being to be idle as long as possible while still taking
942 * advatange of PND's deadline scheduling. The limit of 8
943 * cachelines (used when the FIFO will anyway drain in less time
944 * than 256us) should match what we would be done if trickle
945 * feed were enabled.
946 */
947 return fifo_size - clamp(DIV_ROUND_UP(256 * entries, 64), 0, fifo_size - 8);
948}
949
950static bool vlv_compute_sr_wm(struct drm_device *dev,
951 struct vlv_wm_values *wm)
b445e3b0 952{
ae80152d
VS		 953 struct drm_i915_private *dev_priv = to_i915(dev);
954 struct drm_crtc *crtc;
955 enum pipe pipe = INVALID_PIPE;
956 int num_planes = 0;
957 int fifo_size = 0;
958 struct intel_plane *plane;
b445e3b0 959
ae80152d 960 wm->sr.cursor = wm->sr.plane = 0;
b445e3b0 961
ae80152d
VS		 962 crtc = single_enabled_crtc(dev);
963 /* maxfifo not supported on pipe C */
964 if (crtc && to_intel_crtc(crtc)->pipe != PIPE_C) {
965 pipe = to_intel_crtc(crtc)->pipe;
966 num_planes = !!wm->pipe[pipe].primary +
967 !!wm->pipe[pipe].sprite[0] +
968 !!wm->pipe[pipe].sprite[1];
969 fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
970 }
b445e3b0 971
ae80152d
VS		 972 if (fifo_size == 0 || num_planes > 1)
973 return false;
b445e3b0 974
ae80152d
VS		 975 wm->sr.cursor = vlv_compute_wm(to_intel_crtc(crtc),
976 to_intel_plane(crtc->cursor), 0x3f);
b445e3b0 977
ae80152d
VS		 978 list_for_each_entry(plane, &dev->mode_config.plane_list, base.head) {
979 if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
980 continue;
b445e3b0 981
ae80152d
VS		 982 if (plane->pipe != pipe)
983 continue;
9858425c 984
ae80152d
VS		 985 wm->sr.plane = vlv_compute_wm(to_intel_crtc(crtc),
986 plane, fifo_size);
987 if (wm->sr.plane != 0)
988 break;
989 }
990
991 return true;
b445e3b0
ED		 992}
993
ae80152d 994static void valleyview_update_wm(struct drm_crtc *crtc)
3c2777fd
VS		 995{
996 struct drm_device *dev = crtc->dev;
3c2777fd 997 struct drm_i915_private *dev_priv = dev->dev_private;
ae80152d
VS		 998 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
999 enum pipe pipe = intel_crtc->pipe;
3c2777fd 1000 bool cxsr_enabled;
ae80152d 1001 struct vlv_wm_values wm = dev_priv->wm.vlv;
3c2777fd 1002
ae80152d
VS		 1003 wm.ddl[pipe].primary = vlv_compute_drain_latency(crtc, crtc->primary);
1004 wm.pipe[pipe].primary = vlv_compute_wm(intel_crtc,
1005 to_intel_plane(crtc->primary),
1006 vlv_get_fifo_size(dev, pipe, 0));
3c2777fd 1007
ae80152d
VS		 1008 wm.ddl[pipe].cursor = vlv_compute_drain_latency(crtc, crtc->cursor);
1009 wm.pipe[pipe].cursor = vlv_compute_wm(intel_crtc,
1010 to_intel_plane(crtc->cursor),
1011 0x3f);
3c2777fd 1012
ae80152d 1013 cxsr_enabled = vlv_compute_sr_wm(dev, &wm);
3c2777fd 1014
ae80152d
VS		 1015 if (memcmp(&wm, &dev_priv->wm.vlv, sizeof(wm)) == 0)
1016 return;
3c2777fd 1017
ae80152d
VS		 1018 DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, "
1019 "SR: plane=%d, cursor=%d\n", pipe_name(pipe),
1020 wm.pipe[pipe].primary, wm.pipe[pipe].cursor,
1021 wm.sr.plane, wm.sr.cursor);
3c2777fd 1022
ae80152d
VS		 1023 if (!cxsr_enabled)
1024 intel_set_memory_cxsr(dev_priv, false);
3c2777fd 1025
ae80152d 1026 vlv_write_wm_values(intel_crtc, &wm);
3c2777fd
VS		 1027
1028 if (cxsr_enabled)
1029 intel_set_memory_cxsr(dev_priv, true);
1030}
1031
01e184cc
GB		 1032static void valleyview_update_sprite_wm(struct drm_plane *plane,
1033 struct drm_crtc *crtc,
1034 uint32_t sprite_width,
1035 uint32_t sprite_height,
1036 int pixel_size,
1037 bool enabled, bool scaled)
1038{
1039 struct drm_device *dev = crtc->dev;
1040 struct drm_i915_private *dev_priv = dev->dev_private;
0018fda1
VS		 1041 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1042 enum pipe pipe = intel_crtc->pipe;
01e184cc 1043 int sprite = to_intel_plane(plane)->plane;
ae80152d 1044 bool cxsr_enabled;
0018fda1 1045 struct vlv_wm_values wm = dev_priv->wm.vlv;
01e184cc 1046
ae80152d 1047 if (enabled) {
0018fda1 1048 wm.ddl[pipe].sprite[sprite] =
883a3d2f 1049 vlv_compute_drain_latency(crtc, plane);
ae80152d
VS		 1050
1051 wm.pipe[pipe].sprite[sprite] =
1052 vlv_compute_wm(intel_crtc,
1053 to_intel_plane(plane),
1054 vlv_get_fifo_size(dev, pipe, sprite+1));
1055 } else {
0018fda1 1056 wm.ddl[pipe].sprite[sprite] = 0;
ae80152d
VS		 1057 wm.pipe[pipe].sprite[sprite] = 0;
1058 }
1059
1060 cxsr_enabled = vlv_compute_sr_wm(dev, &wm);
1061
1062 if (memcmp(&wm, &dev_priv->wm.vlv, sizeof(wm)) == 0)
1063 return;
1064
1065 DRM_DEBUG_KMS("Setting FIFO watermarks - %c: sprite %c=%d, "
1066 "SR: plane=%d, cursor=%d\n", pipe_name(pipe),
1067 sprite_name(pipe, sprite),
1068 wm.pipe[pipe].sprite[sprite],
1069 wm.sr.plane, wm.sr.cursor);
1070
1071 if (!cxsr_enabled)
1072 intel_set_memory_cxsr(dev_priv, false);
01e184cc 1073
0018fda1 1074 vlv_write_wm_values(intel_crtc, &wm);
ae80152d
VS		 1075
1076 if (cxsr_enabled)
1077 intel_set_memory_cxsr(dev_priv, true);
01e184cc
GB		 1078}
1079
ae80152d
VS		 1080#define single_plane_enabled(mask) is_power_of_2(mask)
1081
46ba614c 1082static void g4x_update_wm(struct drm_crtc *crtc)
b445e3b0 1083{
46ba614c 1084 struct drm_device *dev = crtc->dev;
b445e3b0
ED		 1085 static const int sr_latency_ns = 12000;
1086 struct drm_i915_private *dev_priv = dev->dev_private;
1087 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1088 int plane_sr, cursor_sr;
1089 unsigned int enabled = 0;
9858425c 1090 bool cxsr_enabled;
b445e3b0 1091
51cea1f4 1092 if (g4x_compute_wm0(dev, PIPE_A,
5aef6003
CW		 1093 &g4x_wm_info, pessimal_latency_ns,
1094 &g4x_cursor_wm_info, pessimal_latency_ns,
b445e3b0 1095 &planea_wm, &cursora_wm))
51cea1f4 1096 enabled |= 1 << PIPE_A;
b445e3b0 1097
51cea1f4 1098 if (g4x_compute_wm0(dev, PIPE_B,
5aef6003
CW		 1099 &g4x_wm_info, pessimal_latency_ns,
1100 &g4x_cursor_wm_info, pessimal_latency_ns,
b445e3b0 1101 &planeb_wm, &cursorb_wm))
51cea1f4 1102 enabled |= 1 << PIPE_B;
b445e3b0 1103
b445e3b0
ED		 1104 if (single_plane_enabled(enabled) &&
1105 g4x_compute_srwm(dev, ffs(enabled) - 1,
1106 sr_latency_ns,
1107 &g4x_wm_info,
1108 &g4x_cursor_wm_info,
52bd02d8 1109 &plane_sr, &cursor_sr)) {
9858425c 1110 cxsr_enabled = true;
52bd02d8 1111 } else {
9858425c 1112 cxsr_enabled = false;
5209b1f4 1113 intel_set_memory_cxsr(dev_priv, false);
52bd02d8
CW		 1114 plane_sr = cursor_sr = 0;
1115 }
b445e3b0 1116
a5043453
VS		 1117 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
1118 "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
b445e3b0
ED		 1119 planea_wm, cursora_wm,
1120 planeb_wm, cursorb_wm,
1121 plane_sr, cursor_sr);
1122
1123 I915_WRITE(DSPFW1,
1124 (plane_sr << DSPFW_SR_SHIFT) |
1125 (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1126 (planeb_wm << DSPFW_PLANEB_SHIFT) |
0a560674 1127 (planea_wm << DSPFW_PLANEA_SHIFT));
b445e3b0 1128 I915_WRITE(DSPFW2,
8c919b28 1129 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
b445e3b0
ED		 1130 (cursora_wm << DSPFW_CURSORA_SHIFT));
1131 /* HPLL off in SR has some issues on G4x... disable it */
1132 I915_WRITE(DSPFW3,
8c919b28 1133 (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
b445e3b0 1134 (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
9858425c
ID		 1135
1136 if (cxsr_enabled)
1137 intel_set_memory_cxsr(dev_priv, true);
b445e3b0
ED		 1138}
1139
46ba614c 1140static void i965_update_wm(struct drm_crtc *unused_crtc)
b445e3b0 1141{
46ba614c 1142 struct drm_device *dev = unused_crtc->dev;
b445e3b0
ED		 1143 struct drm_i915_private *dev_priv = dev->dev_private;
1144 struct drm_crtc *crtc;
1145 int srwm = 1;
1146 int cursor_sr = 16;
9858425c 1147 bool cxsr_enabled;
b445e3b0
ED		 1148
1149 /* Calc sr entries for one plane configs */
1150 crtc = single_enabled_crtc(dev);
1151 if (crtc) {
1152 /* self-refresh has much higher latency */
1153 static const int sr_latency_ns = 12000;
4fe8590a 1154 const struct drm_display_mode *adjusted_mode =
6e3c9717 1155 &to_intel_crtc(crtc)->config->base.adjusted_mode;
241bfc38 1156 int clock = adjusted_mode->crtc_clock;
fec8cba3 1157 int htotal = adjusted_mode->crtc_htotal;
6e3c9717 1158 int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
59bea882 1159 int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
b445e3b0
ED		 1160 unsigned long line_time_us;
1161 int entries;
1162
922044c9 1163 line_time_us = max(htotal * 1000 / clock, 1);
b445e3b0
ED		 1164
1165 /* Use ns/us then divide to preserve precision */
1166 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1167 pixel_size * hdisplay;
1168 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
1169 srwm = I965_FIFO_SIZE - entries;
1170 if (srwm < 0)
1171 srwm = 1;
1172 srwm &= 0x1ff;
1173 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1174 entries, srwm);
1175
1176 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
3dd512fb 1177 pixel_size * crtc->cursor->state->crtc_w;
b445e3b0
ED		 1178 entries = DIV_ROUND_UP(entries,
1179 i965_cursor_wm_info.cacheline_size);
1180 cursor_sr = i965_cursor_wm_info.fifo_size -
1181 (entries + i965_cursor_wm_info.guard_size);
1182
1183 if (cursor_sr > i965_cursor_wm_info.max_wm)
1184 cursor_sr = i965_cursor_wm_info.max_wm;
1185
1186 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1187 "cursor %d\n", srwm, cursor_sr);
1188
9858425c 1189 cxsr_enabled = true;
b445e3b0 1190 } else {
9858425c 1191 cxsr_enabled = false;
b445e3b0 1192 /* Turn off self refresh if both pipes are enabled */
5209b1f4 1193 intel_set_memory_cxsr(dev_priv, false);
b445e3b0
ED		 1194 }
1195
1196 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1197 srwm);
1198
1199 /* 965 has limitations... */
1200 I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
0a560674
VS		 1201 (8 << DSPFW_CURSORB_SHIFT) |
1202 (8 << DSPFW_PLANEB_SHIFT) |
1203 (8 << DSPFW_PLANEA_SHIFT));
1204 I915_WRITE(DSPFW2, (8 << DSPFW_CURSORA_SHIFT) |
1205 (8 << DSPFW_PLANEC_SHIFT_OLD));
b445e3b0
ED		 1206 /* update cursor SR watermark */
1207 I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
9858425c
ID		 1208
1209 if (cxsr_enabled)
1210 intel_set_memory_cxsr(dev_priv, true);
b445e3b0
ED		 1211}
1212
46ba614c 1213static void i9xx_update_wm(struct drm_crtc *unused_crtc)
b445e3b0 1214{
46ba614c 1215 struct drm_device *dev = unused_crtc->dev;
b445e3b0
ED		 1216 struct drm_i915_private *dev_priv = dev->dev_private;
1217 const struct intel_watermark_params *wm_info;
1218 uint32_t fwater_lo;
1219 uint32_t fwater_hi;
1220 int cwm, srwm = 1;
1221 int fifo_size;
1222 int planea_wm, planeb_wm;
1223 struct drm_crtc *crtc, *enabled = NULL;
1224
1225 if (IS_I945GM(dev))
1226 wm_info = &i945_wm_info;
1227 else if (!IS_GEN2(dev))
1228 wm_info = &i915_wm_info;
1229 else
9d539105 1230 wm_info = &i830_a_wm_info;
b445e3b0
ED		 1231
1232 fifo_size = dev_priv->display.get_fifo_size(dev, 0);
1233 crtc = intel_get_crtc_for_plane(dev, 0);
3490ea5d 1234 if (intel_crtc_active(crtc)) {
241bfc38 1235 const struct drm_display_mode *adjusted_mode;
59bea882 1236 int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
b9e0bda3
CW		 1237 if (IS_GEN2(dev))
1238 cpp = 4;
1239
6e3c9717 1240 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
241bfc38 1241 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
b9e0bda3 1242 wm_info, fifo_size, cpp,
5aef6003 1243 pessimal_latency_ns);
b445e3b0 1244 enabled = crtc;
9d539105 1245 } else {
b445e3b0 1246 planea_wm = fifo_size - wm_info->guard_size;
9d539105
VS		 1247 if (planea_wm > (long)wm_info->max_wm)
1248 planea_wm = wm_info->max_wm;
1249 }
1250
1251 if (IS_GEN2(dev))
1252 wm_info = &i830_bc_wm_info;
b445e3b0
ED		 1253
1254 fifo_size = dev_priv->display.get_fifo_size(dev, 1);
1255 crtc = intel_get_crtc_for_plane(dev, 1);
3490ea5d 1256 if (intel_crtc_active(crtc)) {
241bfc38 1257 const struct drm_display_mode *adjusted_mode;
59bea882 1258 int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
b9e0bda3
CW		 1259 if (IS_GEN2(dev))
1260 cpp = 4;
1261
6e3c9717 1262 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
241bfc38 1263 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
b9e0bda3 1264 wm_info, fifo_size, cpp,
5aef6003 1265 pessimal_latency_ns);
b445e3b0
ED		 1266 if (enabled == NULL)
1267 enabled = crtc;
1268 else
1269 enabled = NULL;
9d539105 1270 } else {
b445e3b0 1271 planeb_wm = fifo_size - wm_info->guard_size;
9d539105
VS		 1272 if (planeb_wm > (long)wm_info->max_wm)
1273 planeb_wm = wm_info->max_wm;
1274 }
b445e3b0
ED		 1275
1276 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
1277
2ab1bc9d 1278 if (IS_I915GM(dev) && enabled) {
2ff8fde1 1279 struct drm_i915_gem_object *obj;
2ab1bc9d 1280
59bea882 1281 obj = intel_fb_obj(enabled->primary->state->fb);
2ab1bc9d
DV		 1282
1283 /* self-refresh seems busted with untiled */
2ff8fde1 1284 if (obj->tiling_mode == I915_TILING_NONE)
2ab1bc9d
DV		 1285 enabled = NULL;
1286 }
1287
b445e3b0
ED		 1288 /*
1289 * Overlay gets an aggressive default since video jitter is bad.
1290 */
1291 cwm = 2;
1292
1293 /* Play safe and disable self-refresh before adjusting watermarks. */
5209b1f4 1294 intel_set_memory_cxsr(dev_priv, false);
b445e3b0
ED		 1295
1296 /* Calc sr entries for one plane configs */
1297 if (HAS_FW_BLC(dev) && enabled) {
1298 /* self-refresh has much higher latency */
1299 static const int sr_latency_ns = 6000;
4fe8590a 1300 const struct drm_display_mode *adjusted_mode =
6e3c9717 1301 &to_intel_crtc(enabled)->config->base.adjusted_mode;
241bfc38 1302 int clock = adjusted_mode->crtc_clock;
fec8cba3 1303 int htotal = adjusted_mode->crtc_htotal;
6e3c9717 1304 int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
59bea882 1305 int pixel_size = enabled->primary->state->fb->bits_per_pixel / 8;
b445e3b0
ED		 1306 unsigned long line_time_us;
1307 int entries;
1308
922044c9 1309 line_time_us = max(htotal * 1000 / clock, 1);
b445e3b0
ED		 1310
1311 /* Use ns/us then divide to preserve precision */
1312 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1313 pixel_size * hdisplay;
1314 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
1315 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
1316 srwm = wm_info->fifo_size - entries;
1317 if (srwm < 0)
1318 srwm = 1;
1319
1320 if (IS_I945G(dev) || IS_I945GM(dev))
1321 I915_WRITE(FW_BLC_SELF,
1322 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
1323 else if (IS_I915GM(dev))
1324 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
1325 }
1326
1327 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1328 planea_wm, planeb_wm, cwm, srwm);
1329
1330 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
1331 fwater_hi = (cwm & 0x1f);
1332
1333 /* Set request length to 8 cachelines per fetch */
1334 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
1335 fwater_hi = fwater_hi | (1 << 8);
1336
1337 I915_WRITE(FW_BLC, fwater_lo);
1338 I915_WRITE(FW_BLC2, fwater_hi);
1339
5209b1f4
ID		 1340 if (enabled)
1341 intel_set_memory_cxsr(dev_priv, true);
b445e3b0
ED		 1342}
1343
feb56b93 1344static void i845_update_wm(struct drm_crtc *unused_crtc)
b445e3b0 1345{
46ba614c 1346 struct drm_device *dev = unused_crtc->dev;
b445e3b0
ED		 1347 struct drm_i915_private *dev_priv = dev->dev_private;
1348 struct drm_crtc *crtc;
241bfc38 1349 const struct drm_display_mode *adjusted_mode;
b445e3b0
ED		 1350 uint32_t fwater_lo;
1351 int planea_wm;
1352
1353 crtc = single_enabled_crtc(dev);
1354 if (crtc == NULL)
1355 return;
1356
6e3c9717 1357 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
241bfc38 1358 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
feb56b93 1359 &i845_wm_info,
b445e3b0 1360 dev_priv->display.get_fifo_size(dev, 0),
5aef6003 1361 4, pessimal_latency_ns);
b445e3b0
ED		 1362 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
1363 fwater_lo |= (3<<8) | planea_wm;
1364
1365 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
1366
1367 I915_WRITE(FW_BLC, fwater_lo);
1368}
1369
3658729a
VS		 1370static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
1371 struct drm_crtc *crtc)
801bcfff
PZ		 1372{
1373 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
fd4daa9c 1374 uint32_t pixel_rate;
801bcfff 1375
6e3c9717 1376 pixel_rate = intel_crtc->config->base.adjusted_mode.crtc_clock;
801bcfff
PZ		 1377
1378 /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
1379 * adjust the pixel_rate here. */
1380
6e3c9717 1381 if (intel_crtc->config->pch_pfit.enabled) {
801bcfff 1382 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
6e3c9717 1383 uint32_t pfit_size = intel_crtc->config->pch_pfit.size;
801bcfff 1384
6e3c9717
ACO		 1385 pipe_w = intel_crtc->config->pipe_src_w;
1386 pipe_h = intel_crtc->config->pipe_src_h;
801bcfff
PZ		 1387 pfit_w = (pfit_size >> 16) & 0xFFFF;
1388 pfit_h = pfit_size & 0xFFFF;
1389 if (pipe_w < pfit_w)
1390 pipe_w = pfit_w;
1391 if (pipe_h < pfit_h)
1392 pipe_h = pfit_h;
1393
1394 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
1395 pfit_w * pfit_h);
1396 }
1397
1398 return pixel_rate;
1399}
1400
37126462 1401/* latency must be in 0.1us units. */
23297044 1402static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
801bcfff
PZ		 1403 uint32_t latency)
1404{
1405 uint64_t ret;
1406
3312ba65
VS		 1407 if (WARN(latency == 0, "Latency value missing\n"))
1408 return UINT_MAX;
1409
801bcfff
PZ		 1410 ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
1411 ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
1412
1413 return ret;
1414}
1415
37126462 1416/* latency must be in 0.1us units. */
23297044 1417static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
801bcfff
PZ		 1418 uint32_t horiz_pixels, uint8_t bytes_per_pixel,
1419 uint32_t latency)
1420{
1421 uint32_t ret;
1422
3312ba65
VS		 1423 if (WARN(latency == 0, "Latency value missing\n"))
1424 return UINT_MAX;
1425
801bcfff
PZ		 1426 ret = (latency * pixel_rate) / (pipe_htotal * 10000);
1427 ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
1428 ret = DIV_ROUND_UP(ret, 64) + 2;
1429 return ret;
1430}
1431
23297044 1432static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
cca32e9a
PZ		 1433 uint8_t bytes_per_pixel)
1434{
1435 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
1436}
1437
2ac96d2a
PB		 1438struct skl_pipe_wm_parameters {
1439 bool active;
1440 uint32_t pipe_htotal;
1441 uint32_t pixel_rate; /* in KHz */
1442 struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
1443 struct intel_plane_wm_parameters cursor;
1444};
1445
820c1980 1446struct ilk_pipe_wm_parameters {
801bcfff 1447 bool active;
801bcfff
PZ		 1448 uint32_t pipe_htotal;
1449 uint32_t pixel_rate;
c35426d2
VS		 1450 struct intel_plane_wm_parameters pri;
1451 struct intel_plane_wm_parameters spr;
1452 struct intel_plane_wm_parameters cur;
801bcfff
PZ		 1453};
1454
820c1980 1455struct ilk_wm_maximums {
cca32e9a
PZ		 1456 uint16_t pri;
1457 uint16_t spr;
1458 uint16_t cur;
1459 uint16_t fbc;
1460};
1461
240264f4
VS		 1462/* used in computing the new watermarks state */
1463struct intel_wm_config {
1464 unsigned int num_pipes_active;
1465 bool sprites_enabled;
1466 bool sprites_scaled;
240264f4
VS		 1467};
1468
37126462
VS		 1469/*
1470 * For both WM_PIPE and WM_LP.
1471 * mem_value must be in 0.1us units.
1472 */
820c1980 1473static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
cca32e9a
PZ		 1474 uint32_t mem_value,
1475 bool is_lp)
801bcfff 1476{
cca32e9a
PZ		 1477 uint32_t method1, method2;
1478
c35426d2 1479 if (!params->active || !params->pri.enabled)
801bcfff
PZ		 1480 return 0;
1481
23297044 1482 method1 = ilk_wm_method1(params->pixel_rate,
c35426d2 1483 params->pri.bytes_per_pixel,
cca32e9a
PZ		 1484 mem_value);
1485
1486 if (!is_lp)
1487 return method1;
1488
23297044 1489 method2 = ilk_wm_method2(params->pixel_rate,
cca32e9a 1490 params->pipe_htotal,
c35426d2
VS		 1491 params->pri.horiz_pixels,
1492 params->pri.bytes_per_pixel,
cca32e9a
PZ		 1493 mem_value);
1494
1495 return min(method1, method2);
801bcfff
PZ		 1496}
1497
37126462
VS		 1498/*
1499 * For both WM_PIPE and WM_LP.
1500 * mem_value must be in 0.1us units.
1501 */
820c1980 1502static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
801bcfff
PZ		 1503 uint32_t mem_value)
1504{
1505 uint32_t method1, method2;
1506
c35426d2 1507 if (!params->active || !params->spr.enabled)
801bcfff
PZ		 1508 return 0;
1509
23297044 1510 method1 = ilk_wm_method1(params->pixel_rate,
c35426d2 1511 params->spr.bytes_per_pixel,
801bcfff 1512 mem_value);
23297044 1513 method2 = ilk_wm_method2(params->pixel_rate,
801bcfff 1514 params->pipe_htotal,
c35426d2
VS		 1515 params->spr.horiz_pixels,
1516 params->spr.bytes_per_pixel,
801bcfff
PZ		 1517 mem_value);
1518 return min(method1, method2);
1519}
1520
37126462
VS		 1521/*
1522 * For both WM_PIPE and WM_LP.
1523 * mem_value must be in 0.1us units.
1524 */
820c1980 1525static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
801bcfff
PZ		 1526 uint32_t mem_value)
1527{
c35426d2 1528 if (!params->active || !params->cur.enabled)
801bcfff
PZ		 1529 return 0;
1530
23297044 1531 return ilk_wm_method2(params->pixel_rate,
801bcfff 1532 params->pipe_htotal,
c35426d2
VS		 1533 params->cur.horiz_pixels,
1534 params->cur.bytes_per_pixel,
801bcfff
PZ		 1535 mem_value);
1536}
1537
cca32e9a 1538/* Only for WM_LP. */
820c1980 1539static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
1fda9882 1540 uint32_t pri_val)
cca32e9a 1541{
c35426d2 1542 if (!params->active || !params->pri.enabled)
cca32e9a
PZ		 1543 return 0;
1544
23297044 1545 return ilk_wm_fbc(pri_val,
c35426d2
VS		 1546 params->pri.horiz_pixels,
1547 params->pri.bytes_per_pixel);
cca32e9a
PZ		 1548}
1549
158ae64f
VS		 1550static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
1551{
416f4727
VS		 1552 if (INTEL_INFO(dev)->gen >= 8)
1553 return 3072;
1554 else if (INTEL_INFO(dev)->gen >= 7)
158ae64f
VS		 1555 return 768;
1556 else
1557 return 512;
1558}
1559
4e975081
VS		 1560static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
1561 int level, bool is_sprite)
1562{
1563 if (INTEL_INFO(dev)->gen >= 8)
1564 /* BDW primary/sprite plane watermarks */
1565 return level == 0 ? 255 : 2047;
1566 else if (INTEL_INFO(dev)->gen >= 7)
1567 /* IVB/HSW primary/sprite plane watermarks */
1568 return level == 0 ? 127 : 1023;
1569 else if (!is_sprite)
1570 /* ILK/SNB primary plane watermarks */
1571 return level == 0 ? 127 : 511;
1572 else
1573 /* ILK/SNB sprite plane watermarks */
1574 return level == 0 ? 63 : 255;
1575}
1576
1577static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
1578 int level)
1579{
1580 if (INTEL_INFO(dev)->gen >= 7)
1581 return level == 0 ? 63 : 255;
1582 else
1583 return level == 0 ? 31 : 63;
1584}
1585
1586static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
1587{
1588 if (INTEL_INFO(dev)->gen >= 8)
1589 return 31;
1590 else
1591 return 15;
1592}
1593
158ae64f
VS		 1594/* Calculate the maximum primary/sprite plane watermark */
1595static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
1596 int level,
240264f4 1597 const struct intel_wm_config *config,
158ae64f
VS		 1598 enum intel_ddb_partitioning ddb_partitioning,
1599 bool is_sprite)
1600{
1601 unsigned int fifo_size = ilk_display_fifo_size(dev);
158ae64f
VS		 1602
1603 /* if sprites aren't enabled, sprites get nothing */
240264f4 1604 if (is_sprite && !config->sprites_enabled)
158ae64f
VS		 1605 return 0;
1606
1607 /* HSW allows LP1+ watermarks even with multiple pipes */
240264f4 1608 if (level == 0 || config->num_pipes_active > 1) {
158ae64f
VS		 1609 fifo_size /= INTEL_INFO(dev)->num_pipes;
1610
1611 /*
1612 * For some reason the non self refresh
1613 * FIFO size is only half of the self
1614 * refresh FIFO size on ILK/SNB.
1615 */
1616 if (INTEL_INFO(dev)->gen <= 6)
1617 fifo_size /= 2;
1618 }
1619
240264f4 1620 if (config->sprites_enabled) {
158ae64f
VS		 1621 /* level 0 is always calculated with 1:1 split */
1622 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
1623 if (is_sprite)
1624 fifo_size *= 5;
1625 fifo_size /= 6;
1626 } else {
1627 fifo_size /= 2;
1628 }
1629 }
1630
1631 /* clamp to max that the registers can hold */
4e975081 1632 return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
158ae64f
VS		 1633}
1634
1635/* Calculate the maximum cursor plane watermark */
1636static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
240264f4
VS		 1637 int level,
1638 const struct intel_wm_config *config)
158ae64f
VS		 1639{
1640 /* HSW LP1+ watermarks w/ multiple pipes */
240264f4 1641 if (level > 0 && config->num_pipes_active > 1)
158ae64f
VS		 1642 return 64;
1643
1644 /* otherwise just report max that registers can hold */
4e975081 1645 return ilk_cursor_wm_reg_max(dev, level);
158ae64f
VS		 1646}
1647
d34ff9c6 1648static void ilk_compute_wm_maximums(const struct drm_device *dev,
34982fe1
VS		 1649 int level,
1650 const struct intel_wm_config *config,
1651 enum intel_ddb_partitioning ddb_partitioning,
820c1980 1652 struct ilk_wm_maximums *max)
158ae64f 1653{
240264f4
VS		 1654 max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
1655 max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
1656 max->cur = ilk_cursor_wm_max(dev, level, config);
4e975081 1657 max->fbc = ilk_fbc_wm_reg_max(dev);
158ae64f
VS		 1658}
1659
a3cb4048
VS		 1660static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
1661 int level,
1662 struct ilk_wm_maximums *max)
1663{
1664 max->pri = ilk_plane_wm_reg_max(dev, level, false);
1665 max->spr = ilk_plane_wm_reg_max(dev, level, true);
1666 max->cur = ilk_cursor_wm_reg_max(dev, level);
1667 max->fbc = ilk_fbc_wm_reg_max(dev);
1668}
1669
d9395655 1670static bool ilk_validate_wm_level(int level,
820c1980 1671 const struct ilk_wm_maximums *max,
d9395655 1672 struct intel_wm_level *result)
a9786a11
VS		 1673{
1674 bool ret;
1675
1676 /* already determined to be invalid? */
1677 if (!result->enable)
1678 return false;
1679
1680 result->enable = result->pri_val <= max->pri &&
1681 result->spr_val <= max->spr &&
1682 result->cur_val <= max->cur;
1683
1684 ret = result->enable;
1685
1686 /*
1687 * HACK until we can pre-compute everything,
1688 * and thus fail gracefully if LP0 watermarks
1689 * are exceeded...
1690 */
1691 if (level == 0 && !result->enable) {
1692 if (result->pri_val > max->pri)
1693 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
1694 level, result->pri_val, max->pri);
1695 if (result->spr_val > max->spr)
1696 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
1697 level, result->spr_val, max->spr);
1698 if (result->cur_val > max->cur)
1699 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
1700 level, result->cur_val, max->cur);
1701
1702 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
1703 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
1704 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
1705 result->enable = true;
1706 }
1707
a9786a11
VS		 1708 return ret;
1709}
1710
d34ff9c6 1711static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
6f5ddd17 1712 int level,
820c1980 1713 const struct ilk_pipe_wm_parameters *p,
1fd527cc 1714 struct intel_wm_level *result)
6f5ddd17
VS		 1715{
1716 uint16_t pri_latency = dev_priv->wm.pri_latency[level];
1717 uint16_t spr_latency = dev_priv->wm.spr_latency[level];
1718 uint16_t cur_latency = dev_priv->wm.cur_latency[level];
1719
1720 /* WM1+ latency values stored in 0.5us units */
1721 if (level > 0) {
1722 pri_latency *= 5;
1723 spr_latency *= 5;
1724 cur_latency *= 5;
1725 }
1726
1727 result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
1728 result->spr_val = ilk_compute_spr_wm(p, spr_latency);
1729 result->cur_val = ilk_compute_cur_wm(p, cur_latency);
1730 result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
1731 result->enable = true;
1732}
1733
801bcfff
PZ		 1734static uint32_t
1735hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc)
1f8eeabf
ED		 1736{
1737 struct drm_i915_private *dev_priv = dev->dev_private;
1011d8c4 1738 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6e3c9717 1739 struct drm_display_mode *mode = &intel_crtc->config->base.adjusted_mode;
85a02deb 1740 u32 linetime, ips_linetime;
1f8eeabf 1741
3ef00284 1742 if (!intel_crtc->active)
801bcfff 1743 return 0;
1011d8c4 1744
1f8eeabf
ED		 1745 /* The WM are computed with base on how long it takes to fill a single
1746 * row at the given clock rate, multiplied by 8.
1747 * */
fec8cba3
JB		 1748 linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
1749 mode->crtc_clock);
1750 ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
85a02deb 1751 intel_ddi_get_cdclk_freq(dev_priv));
1f8eeabf 1752
801bcfff
PZ		 1753 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
1754 PIPE_WM_LINETIME_TIME(linetime);
1f8eeabf
ED		 1755}
1756
2af30a5c 1757static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
12b134df
VS		 1758{
1759 struct drm_i915_private *dev_priv = dev->dev_private;
1760
2af30a5c
PB		 1761 if (IS_GEN9(dev)) {
1762 uint32_t val;
4f947386 1763 int ret, i;
367294be 1764 int level, max_level = ilk_wm_max_level(dev);
2af30a5c
PB		 1765
1766 /* read the first set of memory latencies[0:3] */
1767 val = 0; /* data0 to be programmed to 0 for first set */
1768 mutex_lock(&dev_priv->rps.hw_lock);
1769 ret = sandybridge_pcode_read(dev_priv,
1770 GEN9_PCODE_READ_MEM_LATENCY,
1771 &val);
1772 mutex_unlock(&dev_priv->rps.hw_lock);
1773
1774 if (ret) {
1775 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
1776 return;
1777 }
1778
1779 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
1780 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
1781 GEN9_MEM_LATENCY_LEVEL_MASK;
1782 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
1783 GEN9_MEM_LATENCY_LEVEL_MASK;
1784 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
1785 GEN9_MEM_LATENCY_LEVEL_MASK;
1786
1787 /* read the second set of memory latencies[4:7] */
1788 val = 1; /* data0 to be programmed to 1 for second set */
1789 mutex_lock(&dev_priv->rps.hw_lock);
1790 ret = sandybridge_pcode_read(dev_priv,
1791 GEN9_PCODE_READ_MEM_LATENCY,
1792 &val);
1793 mutex_unlock(&dev_priv->rps.hw_lock);
1794 if (ret) {
1795 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
1796 return;
1797 }
1798
1799 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
1800 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
1801 GEN9_MEM_LATENCY_LEVEL_MASK;
1802 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
1803 GEN9_MEM_LATENCY_LEVEL_MASK;
1804 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
1805 GEN9_MEM_LATENCY_LEVEL_MASK;
1806
367294be 1807 /*
6f97235b
DL		 1808 * WaWmMemoryReadLatency:skl
1809 *
367294be
VK		 1810 * punit doesn't take into account the read latency so we need
1811 * to add 2us to the various latency levels we retrieve from
1812 * the punit.
1813 * - W0 is a bit special in that it's the only level that
1814 * can't be disabled if we want to have display working, so
1815 * we always add 2us there.
1816 * - For levels >=1, punit returns 0us latency when they are
1817 * disabled, so we respect that and don't add 2us then
4f947386
VK		 1818 *
1819 * Additionally, if a level n (n > 1) has a 0us latency, all
1820 * levels m (m >= n) need to be disabled. We make sure to
1821 * sanitize the values out of the punit to satisfy this
1822 * requirement.
367294be
VK		 1823 */
1824 wm[0] += 2;
1825 for (level = 1; level <= max_level; level++)
1826 if (wm[level] != 0)
1827 wm[level] += 2;
4f947386
VK		 1828 else {
1829 for (i = level + 1; i <= max_level; i++)
1830 wm[i] = 0;
367294be 1831
4f947386
VK		 1832 break;
1833 }
2af30a5c 1834 } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
12b134df
VS		 1835 uint64_t sskpd = I915_READ64(MCH_SSKPD);
1836
1837 wm[0] = (sskpd >> 56) & 0xFF;
1838 if (wm[0] == 0)
1839 wm[0] = sskpd & 0xF;
e5d5019e
VS		 1840 wm[1] = (sskpd >> 4) & 0xFF;
1841 wm[2] = (sskpd >> 12) & 0xFF;
1842 wm[3] = (sskpd >> 20) & 0x1FF;
1843 wm[4] = (sskpd >> 32) & 0x1FF;
63cf9a13
VS		 1844 } else if (INTEL_INFO(dev)->gen >= 6) {
1845 uint32_t sskpd = I915_READ(MCH_SSKPD);
1846
1847 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
1848 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
1849 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
1850 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
3a88d0ac
VS		 1851 } else if (INTEL_INFO(dev)->gen >= 5) {
1852 uint32_t mltr = I915_READ(MLTR_ILK);
1853
1854 /* ILK primary LP0 latency is 700 ns */
1855 wm[0] = 7;
1856 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
1857 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
12b134df
VS		 1858 }
1859}
1860
53615a5e
VS		 1861static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
1862{
1863 /* ILK sprite LP0 latency is 1300 ns */
1864 if (INTEL_INFO(dev)->gen == 5)
1865 wm[0] = 13;
1866}
1867
1868static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
1869{
1870 /* ILK cursor LP0 latency is 1300 ns */
1871 if (INTEL_INFO(dev)->gen == 5)
1872 wm[0] = 13;
1873
1874 /* WaDoubleCursorLP3Latency:ivb */
1875 if (IS_IVYBRIDGE(dev))
1876 wm[3] *= 2;
1877}
1878
546c81fd 1879int ilk_wm_max_level(const struct drm_device *dev)
26ec971e 1880{
26ec971e 1881 /* how many WM levels are we expecting */
2af30a5c
PB		 1882 if (IS_GEN9(dev))
1883 return 7;
1884 else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
ad0d6dc4 1885 return 4;
26ec971e 1886 else if (INTEL_INFO(dev)->gen >= 6)
ad0d6dc4 1887 return 3;
26ec971e 1888 else
ad0d6dc4
VS		 1889 return 2;
1890}
7526ed79 1891
ad0d6dc4
VS		 1892static void intel_print_wm_latency(struct drm_device *dev,
1893 const char *name,
2af30a5c 1894 const uint16_t wm[8])
ad0d6dc4
VS		 1895{
1896 int level, max_level = ilk_wm_max_level(dev);
26ec971e
VS		 1897
1898 for (level = 0; level <= max_level; level++) {
1899 unsigned int latency = wm[level];
1900
1901 if (latency == 0) {
1902 DRM_ERROR("%s WM%d latency not provided\n",
1903 name, level);
1904 continue;
1905 }
1906
2af30a5c
PB		 1907 /*
1908 * - latencies are in us on gen9.
1909 * - before then, WM1+ latency values are in 0.5us units
1910 */
1911 if (IS_GEN9(dev))
1912 latency *= 10;
1913 else if (level > 0)
26ec971e
VS		 1914 latency *= 5;
1915
1916 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
1917 name, level, wm[level],
1918 latency / 10, latency % 10);
1919 }
1920}
1921
e95a2f75
VS		 1922static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
1923 uint16_t wm[5], uint16_t min)
1924{
1925 int level, max_level = ilk_wm_max_level(dev_priv->dev);
1926
1927 if (wm[0] >= min)
1928 return false;
1929
1930 wm[0] = max(wm[0], min);
1931 for (level = 1; level <= max_level; level++)
1932 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
1933
1934 return true;
1935}
1936
1937static void snb_wm_latency_quirk(struct drm_device *dev)
1938{
1939 struct drm_i915_private *dev_priv = dev->dev_private;
1940 bool changed;
1941
1942 /*
1943 * The BIOS provided WM memory latency values are often
1944 * inadequate for high resolution displays. Adjust them.
1945 */
1946 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
1947 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
1948 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
1949
1950 if (!changed)
1951 return;
1952
1953 DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
1954 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
1955 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
1956 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
1957}
1958
fa50ad61 1959static void ilk_setup_wm_latency(struct drm_device *dev)
53615a5e
VS		 1960{
1961 struct drm_i915_private *dev_priv = dev->dev_private;
1962
1963 intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
1964
1965 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
1966 sizeof(dev_priv->wm.pri_latency));
1967 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
1968 sizeof(dev_priv->wm.pri_latency));
1969
1970 intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
1971 intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
26ec971e
VS		 1972
1973 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
1974 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
1975 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
e95a2f75
VS		 1976
1977 if (IS_GEN6(dev))
1978 snb_wm_latency_quirk(dev);
53615a5e
VS		 1979}
1980
2af30a5c
PB		 1981static void skl_setup_wm_latency(struct drm_device *dev)
1982{
1983 struct drm_i915_private *dev_priv = dev->dev_private;
1984
1985 intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
1986 intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
1987}
1988
820c1980 1989static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
2a44b76b 1990 struct ilk_pipe_wm_parameters *p)
1011d8c4 1991{
7c4a395f
VS		 1992 struct drm_device *dev = crtc->dev;
1993 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1994 enum pipe pipe = intel_crtc->pipe;
7c4a395f 1995 struct drm_plane *plane;
1011d8c4 1996
3ef00284 1997 if (!intel_crtc->active)
2a44b76b 1998 return;
801bcfff 1999
2a44b76b 2000 p->active = true;
6e3c9717 2001 p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
2a44b76b 2002 p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
59bea882 2003 p->pri.bytes_per_pixel = crtc->primary->state->fb->bits_per_pixel / 8;
2a44b76b 2004 p->cur.bytes_per_pixel = 4;
6e3c9717 2005 p->pri.horiz_pixels = intel_crtc->config->pipe_src_w;
3dd512fb 2006 p->cur.horiz_pixels = intel_crtc->base.cursor->state->crtc_w;
2a44b76b
VS		 2007 /* TODO: for now, assume primary and cursor planes are always enabled. */
2008 p->pri.enabled = true;
2009 p->cur.enabled = true;
7c4a395f 2010
af2b653b 2011 drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
801bcfff 2012 struct intel_plane *intel_plane = to_intel_plane(plane);
801bcfff 2013
2a44b76b 2014 if (intel_plane->pipe == pipe) {
7c4a395f 2015 p->spr = intel_plane->wm;
2a44b76b
VS		 2016 break;
2017 }
2018 }
2019}
2020
2021static void ilk_compute_wm_config(struct drm_device *dev,
2022 struct intel_wm_config *config)
2023{
2024 struct intel_crtc *intel_crtc;
2025
2026 /* Compute the currently _active_ config */
d3fcc808 2027 for_each_intel_crtc(dev, intel_crtc) {
2a44b76b 2028 const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
cca32e9a 2029
2a44b76b
VS		 2030 if (!wm->pipe_enabled)
2031 continue;
cca32e9a 2032
2a44b76b
VS		 2033 config->sprites_enabled |= wm->sprites_enabled;
2034 config->sprites_scaled |= wm->sprites_scaled;
2035 config->num_pipes_active++;
cca32e9a 2036 }
801bcfff
PZ		 2037}
2038
0b2ae6d7
VS		 2039/* Compute new watermarks for the pipe */
2040static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
820c1980 2041 const struct ilk_pipe_wm_parameters *params,
0b2ae6d7
VS		 2042 struct intel_pipe_wm *pipe_wm)
2043{
2044 struct drm_device *dev = crtc->dev;
d34ff9c6 2045 const struct drm_i915_private *dev_priv = dev->dev_private;
0b2ae6d7
VS		 2046 int level, max_level = ilk_wm_max_level(dev);
2047 /* LP0 watermark maximums depend on this pipe alone */
2048 struct intel_wm_config config = {
2049 .num_pipes_active = 1,
2050 .sprites_enabled = params->spr.enabled,
2051 .sprites_scaled = params->spr.scaled,
2052 };
820c1980 2053 struct ilk_wm_maximums max;
0b2ae6d7 2054
2a44b76b
VS		 2055 pipe_wm->pipe_enabled = params->active;
2056 pipe_wm->sprites_enabled = params->spr.enabled;
2057 pipe_wm->sprites_scaled = params->spr.scaled;
2058
7b39a0b7
VS		 2059 /* ILK/SNB: LP2+ watermarks only w/o sprites */
2060 if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
2061 max_level = 1;
2062
2063 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2064 if (params->spr.scaled)
2065 max_level = 0;
2066
a3cb4048 2067 ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]);
0b2ae6d7 2068
a42a5719 2069 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
ce0e0713 2070 pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
0b2ae6d7 2071
a3cb4048
VS		 2072 /* LP0 watermarks always use 1/2 DDB partitioning */
2073 ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
2074
0b2ae6d7 2075 /* At least LP0 must be valid */
a3cb4048
VS		 2076 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
2077 return false;
2078
2079 ilk_compute_wm_reg_maximums(dev, 1, &max);
2080
2081 for (level = 1; level <= max_level; level++) {
2082 struct intel_wm_level wm = {};
2083
2084 ilk_compute_wm_level(dev_priv, level, params, &wm);
2085
2086 /*
2087 * Disable any watermark level that exceeds the
2088 * register maximums since such watermarks are
2089 * always invalid.
2090 */
2091 if (!ilk_validate_wm_level(level, &max, &wm))
2092 break;
2093
2094 pipe_wm->wm[level] = wm;
2095 }
2096
2097 return true;
0b2ae6d7
VS		 2098}
2099
2100/*
2101 * Merge the watermarks from all active pipes for a specific level.
2102 */
2103static void ilk_merge_wm_level(struct drm_device *dev,
2104 int level,
2105 struct intel_wm_level *ret_wm)
2106{
2107 const struct intel_crtc *intel_crtc;
2108
d52fea5b
VS		 2109 ret_wm->enable = true;
2110
d3fcc808 2111 for_each_intel_crtc(dev, intel_crtc) {
fe392efd
VS		 2112 const struct intel_pipe_wm *active = &intel_crtc->wm.active;
2113 const struct intel_wm_level *wm = &active->wm[level];
2114
2115 if (!active->pipe_enabled)
2116 continue;
0b2ae6d7 2117
d52fea5b
VS		 2118 /*
2119 * The watermark values may have been used in the past,
2120 * so we must maintain them in the registers for some
2121 * time even if the level is now disabled.
2122 */
0b2ae6d7 2123 if (!wm->enable)
d52fea5b 2124 ret_wm->enable = false;
0b2ae6d7
VS		 2125
2126 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2127 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2128 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2129 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2130 }
0b2ae6d7
VS		 2131}
2132
2133/*
2134 * Merge all low power watermarks for all active pipes.
2135 */
2136static void ilk_wm_merge(struct drm_device *dev,
0ba22e26 2137 const struct intel_wm_config *config,
820c1980 2138 const struct ilk_wm_maximums *max,
0b2ae6d7
VS		 2139 struct intel_pipe_wm *merged)
2140{
2141 int level, max_level = ilk_wm_max_level(dev);
d52fea5b 2142 int last_enabled_level = max_level;
0b2ae6d7 2143
0ba22e26
VS		 2144 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
2145 if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
2146 config->num_pipes_active > 1)
2147 return;
2148
6c8b6c28
VS		 2149 /* ILK: FBC WM must be disabled always */
2150 merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
0b2ae6d7
VS		 2151
2152 /* merge each WM1+ level */
2153 for (level = 1; level <= max_level; level++) {
2154 struct intel_wm_level *wm = &merged->wm[level];
2155
2156 ilk_merge_wm_level(dev, level, wm);
2157
d52fea5b
VS		 2158 if (level > last_enabled_level)
2159 wm->enable = false;
2160 else if (!ilk_validate_wm_level(level, max, wm))
2161 /* make sure all following levels get disabled */
2162 last_enabled_level = level - 1;
0b2ae6d7
VS		 2163
2164 /*
2165 * The spec says it is preferred to disable
2166 * FBC WMs instead of disabling a WM level.
2167 */
2168 if (wm->fbc_val > max->fbc) {
d52fea5b
VS		 2169 if (wm->enable)
2170 merged->fbc_wm_enabled = false;
0b2ae6d7
VS		 2171 wm->fbc_val = 0;
2172 }
2173 }
6c8b6c28
VS		 2174
2175 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
2176 /*
2177 * FIXME this is racy. FBC might get enabled later.
2178 * What we should check here is whether FBC can be
2179 * enabled sometime later.
2180 */
2181 if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) {
2182 for (level = 2; level <= max_level; level++) {
2183 struct intel_wm_level *wm = &merged->wm[level];
2184
2185 wm->enable = false;
2186 }
2187 }
0b2ae6d7
VS		 2188}
2189
b380ca3c
VS		 2190static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
2191{
2192 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
2193 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
2194}
2195
a68d68ee
VS		 2196/* The value we need to program into the WM_LPx latency field */
2197static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
2198{
2199 struct drm_i915_private *dev_priv = dev->dev_private;
2200
a42a5719 2201 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
a68d68ee
VS		 2202 return 2 * level;
2203 else
2204 return dev_priv->wm.pri_latency[level];
2205}
2206
820c1980 2207static void ilk_compute_wm_results(struct drm_device *dev,
0362c781 2208 const struct intel_pipe_wm *merged,
609cedef 2209 enum intel_ddb_partitioning partitioning,
820c1980 2210 struct ilk_wm_values *results)
801bcfff 2211{
0b2ae6d7
VS		 2212 struct intel_crtc *intel_crtc;
2213 int level, wm_lp;
cca32e9a 2214
0362c781 2215 results->enable_fbc_wm = merged->fbc_wm_enabled;
609cedef 2216 results->partitioning = partitioning;
cca32e9a 2217
0b2ae6d7 2218 /* LP1+ register values */
cca32e9a 2219 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
1fd527cc 2220 const struct intel_wm_level *r;
801bcfff 2221
b380ca3c 2222 level = ilk_wm_lp_to_level(wm_lp, merged);
0b2ae6d7 2223
0362c781 2224 r = &merged->wm[level];
cca32e9a 2225
d52fea5b
VS		 2226 /*
2227 * Maintain the watermark values even if the level is
2228 * disabled. Doing otherwise could cause underruns.
2229 */
2230 results->wm_lp[wm_lp - 1] =
a68d68ee 2231 (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
416f4727
VS		 2232 (r->pri_val << WM1_LP_SR_SHIFT) |
2233 r->cur_val;
2234
d52fea5b
VS		 2235 if (r->enable)
2236 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
2237
416f4727
VS		 2238 if (INTEL_INFO(dev)->gen >= 8)
2239 results->wm_lp[wm_lp - 1] |=
2240 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
2241 else
2242 results->wm_lp[wm_lp - 1] |=
2243 r->fbc_val << WM1_LP_FBC_SHIFT;
2244
d52fea5b
VS		 2245 /*
2246 * Always set WM1S_LP_EN when spr_val != 0, even if the
2247 * level is disabled. Doing otherwise could cause underruns.
2248 */
6cef2b8a
VS		 2249 if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
2250 WARN_ON(wm_lp != 1);
2251 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
2252 } else
2253 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
cca32e9a 2254 }
801bcfff 2255
0b2ae6d7 2256 /* LP0 register values */
d3fcc808 2257 for_each_intel_crtc(dev, intel_crtc) {
0b2ae6d7
VS		 2258 enum pipe pipe = intel_crtc->pipe;
2259 const struct intel_wm_level *r =
2260 &intel_crtc->wm.active.wm[0];
2261
2262 if (WARN_ON(!r->enable))
2263 continue;
2264
2265 results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
1011d8c4 2266
0b2ae6d7
VS		 2267 results->wm_pipe[pipe] =
2268 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
2269 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
2270 r->cur_val;
801bcfff
PZ		 2271 }
2272}
2273
861f3389
PZ		 2274/* Find the result with the highest level enabled. Check for enable_fbc_wm in
2275 * case both are at the same level. Prefer r1 in case they're the same. */
820c1980 2276static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
198a1e9b
VS		 2277 struct intel_pipe_wm *r1,
2278 struct intel_pipe_wm *r2)
861f3389 2279{
198a1e9b
VS		 2280 int level, max_level = ilk_wm_max_level(dev);
2281 int level1 = 0, level2 = 0;
861f3389 2282
198a1e9b
VS		 2283 for (level = 1; level <= max_level; level++) {
2284 if (r1->wm[level].enable)
2285 level1 = level;
2286 if (r2->wm[level].enable)
2287 level2 = level;
861f3389
PZ		 2288 }
2289
198a1e9b
VS		 2290 if (level1 == level2) {
2291 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
861f3389
PZ		 2292 return r2;
2293 else
2294 return r1;
198a1e9b 2295 } else if (level1 > level2) {
861f3389
PZ		 2296 return r1;
2297 } else {
2298 return r2;
2299 }
2300}
2301
49a687c4
VS		 2302/* dirty bits used to track which watermarks need changes */
2303#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
2304#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
2305#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
2306#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
2307#define WM_DIRTY_FBC (1 << 24)
2308#define WM_DIRTY_DDB (1 << 25)
2309
055e393f 2310static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
820c1980
ID		 2311 const struct ilk_wm_values *old,
2312 const struct ilk_wm_values *new)
49a687c4
VS		 2313{
2314 unsigned int dirty = 0;
2315 enum pipe pipe;
2316 int wm_lp;
2317
055e393f 2318 for_each_pipe(dev_priv, pipe) {
49a687c4
VS		 2319 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
2320 dirty |= WM_DIRTY_LINETIME(pipe);
2321 /* Must disable LP1+ watermarks too */
2322 dirty |= WM_DIRTY_LP_ALL;
2323 }
2324
2325 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
2326 dirty |= WM_DIRTY_PIPE(pipe);
2327 /* Must disable LP1+ watermarks too */
2328 dirty |= WM_DIRTY_LP_ALL;
2329 }
2330 }
2331
2332 if (old->enable_fbc_wm != new->enable_fbc_wm) {
2333 dirty |= WM_DIRTY_FBC;
2334 /* Must disable LP1+ watermarks too */
2335 dirty |= WM_DIRTY_LP_ALL;
2336 }
2337
2338 if (old->partitioning != new->partitioning) {
2339 dirty |= WM_DIRTY_DDB;
2340 /* Must disable LP1+ watermarks too */
2341 dirty |= WM_DIRTY_LP_ALL;
2342 }
2343
2344 /* LP1+ watermarks already deemed dirty, no need to continue */
2345 if (dirty & WM_DIRTY_LP_ALL)
2346 return dirty;
2347
2348 /* Find the lowest numbered LP1+ watermark in need of an update... */
2349 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2350 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
2351 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
2352 break;
2353 }
2354
2355 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
2356 for (; wm_lp <= 3; wm_lp++)
2357 dirty |= WM_DIRTY_LP(wm_lp);
2358
2359 return dirty;
2360}
2361
8553c18e
VS		 2362static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
2363 unsigned int dirty)
801bcfff 2364{
820c1980 2365 struct ilk_wm_values *previous = &dev_priv->wm.hw;
8553c18e 2366 bool changed = false;
801bcfff 2367
facd619b
VS		 2368 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
2369 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
2370 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
8553c18e 2371 changed = true;
facd619b
VS		 2372 }
2373 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
2374 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
2375 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
8553c18e 2376 changed = true;
facd619b
VS		 2377 }
2378 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
2379 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
2380 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
8553c18e 2381 changed = true;
facd619b 2382 }
801bcfff 2383
facd619b
VS		 2384 /*
2385 * Don't touch WM1S_LP_EN here.
2386 * Doing so could cause underruns.
2387 */
6cef2b8a 2388
8553c18e
VS		 2389 return changed;
2390}
2391
2392/*
2393 * The spec says we shouldn't write when we don't need, because every write
2394 * causes WMs to be re-evaluated, expending some power.
2395 */
820c1980
ID		 2396static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2397 struct ilk_wm_values *results)
8553c18e
VS		 2398{
2399 struct drm_device *dev = dev_priv->dev;
820c1980 2400 struct ilk_wm_values *previous = &dev_priv->wm.hw;
8553c18e
VS		 2401 unsigned int dirty;
2402 uint32_t val;
2403
055e393f 2404 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
8553c18e
VS		 2405 if (!dirty)
2406 return;
2407
2408 _ilk_disable_lp_wm(dev_priv, dirty);
2409
49a687c4 2410 if (dirty & WM_DIRTY_PIPE(PIPE_A))
801bcfff 2411 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
49a687c4 2412 if (dirty & WM_DIRTY_PIPE(PIPE_B))
801bcfff 2413 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
49a687c4 2414 if (dirty & WM_DIRTY_PIPE(PIPE_C))
801bcfff
PZ		 2415 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
2416
49a687c4 2417 if (dirty & WM_DIRTY_LINETIME(PIPE_A))
801bcfff 2418 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
49a687c4 2419 if (dirty & WM_DIRTY_LINETIME(PIPE_B))
801bcfff 2420 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
49a687c4 2421 if (dirty & WM_DIRTY_LINETIME(PIPE_C))
801bcfff
PZ		 2422 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
2423
49a687c4 2424 if (dirty & WM_DIRTY_DDB) {
a42a5719 2425 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
ac9545fd
VS		 2426 val = I915_READ(WM_MISC);
2427 if (results->partitioning == INTEL_DDB_PART_1_2)
2428 val &= ~WM_MISC_DATA_PARTITION_5_6;
2429 else
2430 val |= WM_MISC_DATA_PARTITION_5_6;
2431 I915_WRITE(WM_MISC, val);
2432 } else {
2433 val = I915_READ(DISP_ARB_CTL2);
2434 if (results->partitioning == INTEL_DDB_PART_1_2)
2435 val &= ~DISP_DATA_PARTITION_5_6;
2436 else
2437 val |= DISP_DATA_PARTITION_5_6;
2438 I915_WRITE(DISP_ARB_CTL2, val);
2439 }
1011d8c4
PZ		 2440 }
2441
49a687c4 2442 if (dirty & WM_DIRTY_FBC) {
cca32e9a
PZ		 2443 val = I915_READ(DISP_ARB_CTL);
2444 if (results->enable_fbc_wm)
2445 val &= ~DISP_FBC_WM_DIS;
2446 else
2447 val |= DISP_FBC_WM_DIS;
2448 I915_WRITE(DISP_ARB_CTL, val);
2449 }
2450
954911eb
ID		 2451 if (dirty & WM_DIRTY_LP(1) &&
2452 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
2453 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
2454
2455 if (INTEL_INFO(dev)->gen >= 7) {
6cef2b8a
VS		 2456 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
2457 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
2458 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
2459 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
2460 }
801bcfff 2461
facd619b 2462 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
801bcfff 2463 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
facd619b 2464 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
801bcfff 2465 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
facd619b 2466 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
801bcfff 2467 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
609cedef
VS		 2468
2469 dev_priv->wm.hw = *results;
801bcfff
PZ		 2470}
2471
8553c18e
VS		 2472static bool ilk_disable_lp_wm(struct drm_device *dev)
2473{
2474 struct drm_i915_private *dev_priv = dev->dev_private;
2475
2476 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
2477}
2478
b9cec075
DL		 2479/*
2480 * On gen9, we need to allocate Display Data Buffer (DDB) portions to the
2481 * different active planes.
2482 */
2483
2484#define SKL_DDB_SIZE 896 /* in blocks */
2485
2486static void
2487skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
2488 struct drm_crtc *for_crtc,
2489 const struct intel_wm_config *config,
2490 const struct skl_pipe_wm_parameters *params,
2491 struct skl_ddb_entry *alloc /* out */)
2492{
2493 struct drm_crtc *crtc;
2494 unsigned int pipe_size, ddb_size;
2495 int nth_active_pipe;
2496
2497 if (!params->active) {
2498 alloc->start = 0;
2499 alloc->end = 0;
2500 return;
2501 }
2502
2503 ddb_size = SKL_DDB_SIZE;
2504
2505 ddb_size -= 4; /* 4 blocks for bypass path allocation */
2506
2507 nth_active_pipe = 0;
2508 for_each_crtc(dev, crtc) {
3ef00284 2509 if (!to_intel_crtc(crtc)->active)
b9cec075
DL		 2510 continue;
2511
2512 if (crtc == for_crtc)
2513 break;
2514
2515 nth_active_pipe++;
2516 }
2517
2518 pipe_size = ddb_size / config->num_pipes_active;
2519 alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
16160e3d 2520 alloc->end = alloc->start + pipe_size;
b9cec075
DL		 2521}
2522
2523static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
2524{
2525 if (config->num_pipes_active == 1)
2526 return 32;
2527
2528 return 8;
2529}
2530
a269c583
DL		 2531static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
2532{
2533 entry->start = reg & 0x3ff;
2534 entry->end = (reg >> 16) & 0x3ff;
16160e3d
DL		 2535 if (entry->end)
2536 entry->end += 1;
a269c583
DL		 2537}
2538
08db6652
DL		 2539void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
2540 struct skl_ddb_allocation *ddb /* out */)
a269c583 2541{
a269c583
DL		 2542 enum pipe pipe;
2543 int plane;
2544 u32 val;
2545
2546 for_each_pipe(dev_priv, pipe) {
dd740780 2547 for_each_plane(dev_priv, pipe, plane) {
a269c583
DL		 2548 val = I915_READ(PLANE_BUF_CFG(pipe, plane));
2549 skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
2550 val);
2551 }
2552
2553 val = I915_READ(CUR_BUF_CFG(pipe));
2554 skl_ddb_entry_init_from_hw(&ddb->cursor[pipe], val);
2555 }
2556}
2557
b9cec075
DL		 2558static unsigned int
2559skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p)
2560{
2561 return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel;
2562}
2563
2564/*
2565 * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
2566 * a 8192x4096@32bpp framebuffer:
2567 * 3 * 4096 * 8192 * 4 < 2^32
2568 */
2569static unsigned int
2570skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc,
2571 const struct skl_pipe_wm_parameters *params)
2572{
2573 unsigned int total_data_rate = 0;
2574 int plane;
2575
2576 for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
2577 const struct intel_plane_wm_parameters *p;
2578
2579 p = &params->plane[plane];
2580 if (!p->enabled)
2581 continue;
2582
2583 total_data_rate += skl_plane_relative_data_rate(p);
2584 }
2585
2586 return total_data_rate;
2587}
2588
2589static void
2590skl_allocate_pipe_ddb(struct drm_crtc *crtc,
2591 const struct intel_wm_config *config,
2592 const struct skl_pipe_wm_parameters *params,
2593 struct skl_ddb_allocation *ddb /* out */)
2594{
2595 struct drm_device *dev = crtc->dev;
dd740780 2596 struct drm_i915_private *dev_priv = dev->dev_private;
b9cec075
DL		 2597 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2598 enum pipe pipe = intel_crtc->pipe;
34bb56af 2599 struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
b9cec075 2600 uint16_t alloc_size, start, cursor_blocks;
80958155 2601 uint16_t minimum[I915_MAX_PLANES];
b9cec075
DL		 2602 unsigned int total_data_rate;
2603 int plane;
2604
34bb56af
DL		 2605 skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc);
2606 alloc_size = skl_ddb_entry_size(alloc);
b9cec075
DL		 2607 if (alloc_size == 0) {
2608 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
2609 memset(&ddb->cursor[pipe], 0, sizeof(ddb->cursor[pipe]));
2610 return;
2611 }
2612
2613 cursor_blocks = skl_cursor_allocation(config);
34bb56af
DL		 2614 ddb->cursor[pipe].start = alloc->end - cursor_blocks;
2615 ddb->cursor[pipe].end = alloc->end;
b9cec075
DL		 2616
2617 alloc_size -= cursor_blocks;
34bb56af 2618 alloc->end -= cursor_blocks;
b9cec075 2619
80958155 2620 /* 1. Allocate the mininum required blocks for each active plane */
dd740780 2621 for_each_plane(dev_priv, pipe, plane) {
80958155
DL		 2622 const struct intel_plane_wm_parameters *p;
2623
2624 p = &params->plane[plane];
2625 if (!p->enabled)
2626 continue;
2627
2628 minimum[plane] = 8;
2629 alloc_size -= minimum[plane];
2630 }
2631
b9cec075 2632 /*
80958155
DL		 2633 * 2. Distribute the remaining space in proportion to the amount of
2634 * data each plane needs to fetch from memory.
b9cec075
DL		 2635 *
2636 * FIXME: we may not allocate every single block here.
2637 */
2638 total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params);
2639
34bb56af 2640 start = alloc->start;
b9cec075
DL		 2641 for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
2642 const struct intel_plane_wm_parameters *p;
2643 unsigned int data_rate;
2644 uint16_t plane_blocks;
2645
2646 p = &params->plane[plane];
2647 if (!p->enabled)
2648 continue;
2649
2650 data_rate = skl_plane_relative_data_rate(p);
2651
2652 /*
2653 * promote the expression to 64 bits to avoid overflowing, the
2654 * result is < available as data_rate / total_data_rate < 1
2655 */
80958155
DL		 2656 plane_blocks = minimum[plane];
2657 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
2658 total_data_rate);
b9cec075
DL		 2659
2660 ddb->plane[pipe][plane].start = start;
16160e3d 2661 ddb->plane[pipe][plane].end = start + plane_blocks;
b9cec075
DL		 2662
2663 start += plane_blocks;
2664 }
2665
2666}
2667
5cec258b 2668static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
2d41c0b5
PB		 2669{
2670 /* TODO: Take into account the scalers once we support them */
2d112de7 2671 return config->base.adjusted_mode.crtc_clock;
2d41c0b5
PB		 2672}
2673
2674/*
2675 * The max latency should be 257 (max the punit can code is 255 and we add 2us
2676 * for the read latency) and bytes_per_pixel should always be <= 8, so that
2677 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
2678 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
2679*/
2680static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
2681 uint32_t latency)
2682{
2683 uint32_t wm_intermediate_val, ret;
2684
2685 if (latency == 0)
2686 return UINT_MAX;
2687
d4c2aa60 2688 wm_intermediate_val = latency * pixel_rate * bytes_per_pixel / 512;
2d41c0b5
PB		 2689 ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
2690
2691 return ret;
2692}
2693
2694static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
2695 uint32_t horiz_pixels, uint8_t bytes_per_pixel,
0fda6568 2696 uint64_t tiling, uint32_t latency)
2d41c0b5 2697{
d4c2aa60
TU		 2698 uint32_t ret;
2699 uint32_t plane_bytes_per_line, plane_blocks_per_line;
2700 uint32_t wm_intermediate_val;
2d41c0b5
PB		 2701
2702 if (latency == 0)
2703 return UINT_MAX;
2704
2705 plane_bytes_per_line = horiz_pixels * bytes_per_pixel;
0fda6568
TU		 2706
2707 if (tiling == I915_FORMAT_MOD_Y_TILED ||
2708 tiling == I915_FORMAT_MOD_Yf_TILED) {
2709 plane_bytes_per_line *= 4;
2710 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
2711 plane_blocks_per_line /= 4;
2712 } else {
2713 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
2714 }
2715
2d41c0b5
PB		 2716 wm_intermediate_val = latency * pixel_rate;
2717 ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
d4c2aa60 2718 plane_blocks_per_line;
2d41c0b5
PB		 2719
2720 return ret;
2721}
2722
2d41c0b5
PB		 2723static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
2724 const struct intel_crtc *intel_crtc)
2725{
2726 struct drm_device *dev = intel_crtc->base.dev;
2727 struct drm_i915_private *dev_priv = dev->dev_private;
2728 const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
2729 enum pipe pipe = intel_crtc->pipe;
2730
2731 if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe],
2732 sizeof(new_ddb->plane[pipe])))
2733 return true;
2734
2735 if (memcmp(&new_ddb->cursor[pipe], &cur_ddb->cursor[pipe],
2736 sizeof(new_ddb->cursor[pipe])))
2737 return true;
2738
2739 return false;
2740}
2741
2742static void skl_compute_wm_global_parameters(struct drm_device *dev,
2743 struct intel_wm_config *config)
2744{
2745 struct drm_crtc *crtc;
2746 struct drm_plane *plane;
2747
2748 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
3ef00284 2749 config->num_pipes_active += to_intel_crtc(crtc)->active;
2d41c0b5
PB		 2750
2751 /* FIXME: I don't think we need those two global parameters on SKL */
2752 list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
2753 struct intel_plane *intel_plane = to_intel_plane(plane);
2754
2755 config->sprites_enabled |= intel_plane->wm.enabled;
2756 config->sprites_scaled |= intel_plane->wm.scaled;
2757 }
2758}
2759
2760static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc,
2761 struct skl_pipe_wm_parameters *p)
2762{
2763 struct drm_device *dev = crtc->dev;
2764 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2765 enum pipe pipe = intel_crtc->pipe;
2766 struct drm_plane *plane;
0fda6568 2767 struct drm_framebuffer *fb;
2d41c0b5
PB		 2768 int i = 1; /* Index for sprite planes start */
2769
3ef00284 2770 p->active = intel_crtc->active;
2d41c0b5 2771 if (p->active) {
6e3c9717
ACO		 2772 p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
2773 p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config);
2d41c0b5
PB		 2774
2775 /*
2776 * For now, assume primary and cursor planes are always enabled.
2777 */
2778 p->plane[0].enabled = true;
2779 p->plane[0].bytes_per_pixel =
59bea882 2780 crtc->primary->state->fb->bits_per_pixel / 8;
6e3c9717
ACO		 2781 p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w;
2782 p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h;
0fda6568
TU		 2783 p->plane[0].tiling = DRM_FORMAT_MOD_NONE;
2784 fb = crtc->primary->state->fb;
2785 /*
2786 * Framebuffer can be NULL on plane disable, but it does not
2787 * matter for watermarks if we assume no tiling in that case.
2788 */
2789 if (fb)
2790 p->plane[0].tiling = fb->modifier[0];
2d41c0b5
PB		 2791
2792 p->cursor.enabled = true;
2793 p->cursor.bytes_per_pixel = 4;
3dd512fb
MR		 2794 p->cursor.horiz_pixels = intel_crtc->base.cursor->state->crtc_w ?
2795 intel_crtc->base.cursor->state->crtc_w : 64;
2d41c0b5
PB		 2796 }
2797
2798 list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
2799 struct intel_plane *intel_plane = to_intel_plane(plane);
2800
a712f8eb
SJ		 2801 if (intel_plane->pipe == pipe &&
2802 plane->type == DRM_PLANE_TYPE_OVERLAY)
2d41c0b5
PB		 2803 p->plane[i++] = intel_plane->wm;
2804 }
2805}
2806
d4c2aa60
TU		 2807static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
2808 struct skl_pipe_wm_parameters *p,
afb024aa
DL		 2809 struct intel_plane_wm_parameters *p_params,
2810 uint16_t ddb_allocation,
d4c2aa60 2811 int level,
afb024aa
DL		 2812 uint16_t *out_blocks, /* out */
2813 uint8_t *out_lines /* out */)
2d41c0b5 2814{
d4c2aa60
TU		 2815 uint32_t latency = dev_priv->wm.skl_latency[level];
2816 uint32_t method1, method2;
2817 uint32_t plane_bytes_per_line, plane_blocks_per_line;
2818 uint32_t res_blocks, res_lines;
2819 uint32_t selected_result;
2d41c0b5 2820
d4c2aa60 2821 if (latency == 0 || !p->active || !p_params->enabled)
2d41c0b5
PB		 2822 return false;
2823
2824 method1 = skl_wm_method1(p->pixel_rate,
2825 p_params->bytes_per_pixel,
d4c2aa60 2826 latency);
2d41c0b5
PB		 2827 method2 = skl_wm_method2(p->pixel_rate,
2828 p->pipe_htotal,
2829 p_params->horiz_pixels,
2830 p_params->bytes_per_pixel,
0fda6568 2831 p_params->tiling,
d4c2aa60 2832 latency);
2d41c0b5
PB		 2833
2834 plane_bytes_per_line = p_params->horiz_pixels *
2835 p_params->bytes_per_pixel;
d4c2aa60 2836 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
2d41c0b5 2837
0fda6568
TU		 2838 if (p_params->tiling == I915_FORMAT_MOD_Y_TILED ||
2839 p_params->tiling == I915_FORMAT_MOD_Yf_TILED) {
2840 uint32_t y_tile_minimum = plane_blocks_per_line * 4;
2841 selected_result = max(method2, y_tile_minimum);
2842 } else {
2843 if ((ddb_allocation / plane_blocks_per_line) >= 1)
2844 selected_result = min(method1, method2);
2845 else
2846 selected_result = method1;
2847 }
2d41c0b5 2848
d4c2aa60
TU		 2849 res_blocks = selected_result + 1;
2850 res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line);
e6d66171 2851
0fda6568
TU		 2852 if (level >= 1 && level <= 7) {
2853 if (p_params->tiling == I915_FORMAT_MOD_Y_TILED ||
2854 p_params->tiling == I915_FORMAT_MOD_Yf_TILED)
2855 res_lines += 4;
2856 else
2857 res_blocks++;
2858 }
e6d66171 2859
d4c2aa60 2860 if (res_blocks >= ddb_allocation || res_lines > 31)
e6d66171
DL		 2861 return false;
2862
2863 *out_blocks = res_blocks;
2864 *out_lines = res_lines;
2d41c0b5
PB		 2865
2866 return true;
2867}
2868
2869static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
2870 struct skl_ddb_allocation *ddb,
2871 struct skl_pipe_wm_parameters *p,
2872 enum pipe pipe,
2873 int level,
2874 int num_planes,
2875 struct skl_wm_level *result)
2876{
2d41c0b5
PB		 2877 uint16_t ddb_blocks;
2878 int i;
2879
2880 for (i = 0; i < num_planes; i++) {
2881 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
2882
d4c2aa60
TU		 2883 result->plane_en[i] = skl_compute_plane_wm(dev_priv,
2884 p, &p->plane[i],
2d41c0b5 2885 ddb_blocks,
d4c2aa60 2886 level,
2d41c0b5
PB		 2887 &result->plane_res_b[i],
2888 &result->plane_res_l[i]);
2889 }
2890
2891 ddb_blocks = skl_ddb_entry_size(&ddb->cursor[pipe]);
d4c2aa60
TU		 2892 result->cursor_en = skl_compute_plane_wm(dev_priv, p, &p->cursor,
2893 ddb_blocks, level,
2894 &result->cursor_res_b,
2d41c0b5
PB		 2895 &result->cursor_res_l);
2896}
2897
407b50f3
DL		 2898static uint32_t
2899skl_compute_linetime_wm(struct drm_crtc *crtc, struct skl_pipe_wm_parameters *p)
2900{
3ef00284 2901 if (!to_intel_crtc(crtc)->active)
407b50f3
DL		 2902 return 0;
2903
2904 return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate);
2905
2906}
2907
2908static void skl_compute_transition_wm(struct drm_crtc *crtc,
2909 struct skl_pipe_wm_parameters *params,
9414f563 2910 struct skl_wm_level *trans_wm /* out */)
407b50f3 2911{
9414f563
DL		 2912 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2913 int i;
2914
407b50f3
DL		 2915 if (!params->active)
2916 return;
9414f563
DL		 2917
2918 /* Until we know more, just disable transition WMs */
2919 for (i = 0; i < intel_num_planes(intel_crtc); i++)
2920 trans_wm->plane_en[i] = false;
2921 trans_wm->cursor_en = false;
407b50f3
DL		 2922}
2923
2d41c0b5
PB		 2924static void skl_compute_pipe_wm(struct drm_crtc *crtc,
2925 struct skl_ddb_allocation *ddb,
2926 struct skl_pipe_wm_parameters *params,
2927 struct skl_pipe_wm *pipe_wm)
2928{
2929 struct drm_device *dev = crtc->dev;
2930 const struct drm_i915_private *dev_priv = dev->dev_private;
2931 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2932 int level, max_level = ilk_wm_max_level(dev);
2933
2934 for (level = 0; level <= max_level; level++) {
2935 skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe,
2936 level, intel_num_planes(intel_crtc),
2937 &pipe_wm->wm[level]);
2938 }
2939 pipe_wm->linetime = skl_compute_linetime_wm(crtc, params);
2940
9414f563 2941 skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm);
2d41c0b5
PB		 2942}
2943
2944static void skl_compute_wm_results(struct drm_device *dev,
2945 struct skl_pipe_wm_parameters *p,
2946 struct skl_pipe_wm *p_wm,
2947 struct skl_wm_values *r,
2948 struct intel_crtc *intel_crtc)
2949{
2950 int level, max_level = ilk_wm_max_level(dev);
2951 enum pipe pipe = intel_crtc->pipe;
9414f563
DL		 2952 uint32_t temp;
2953 int i;
2d41c0b5
PB		 2954
2955 for (level = 0; level <= max_level; level++) {
2d41c0b5
PB		 2956 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
2957 temp = 0;
2d41c0b5
PB		 2958
2959 temp |= p_wm->wm[level].plane_res_l[i] <<
2960 PLANE_WM_LINES_SHIFT;
2961 temp |= p_wm->wm[level].plane_res_b[i];
2962 if (p_wm->wm[level].plane_en[i])
2963 temp |= PLANE_WM_EN;
2964
2965 r->plane[pipe][i][level] = temp;
2d41c0b5
PB		 2966 }
2967
2968 temp = 0;
2d41c0b5
PB		 2969
2970 temp |= p_wm->wm[level].cursor_res_l << PLANE_WM_LINES_SHIFT;
2971 temp |= p_wm->wm[level].cursor_res_b;
2972
2973 if (p_wm->wm[level].cursor_en)
2974 temp |= PLANE_WM_EN;
2975
2976 r->cursor[pipe][level] = temp;
2d41c0b5
PB		 2977
2978 }
2979
9414f563
DL		 2980 /* transition WMs */
2981 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
2982 temp = 0;
2983 temp |= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
2984 temp |= p_wm->trans_wm.plane_res_b[i];
2985 if (p_wm->trans_wm.plane_en[i])
2986 temp |= PLANE_WM_EN;
2987
2988 r->plane_trans[pipe][i] = temp;
2989 }
2990
2991 temp = 0;
2992 temp |= p_wm->trans_wm.cursor_res_l << PLANE_WM_LINES_SHIFT;
2993 temp |= p_wm->trans_wm.cursor_res_b;
2994 if (p_wm->trans_wm.cursor_en)
2995 temp |= PLANE_WM_EN;
2996
2997 r->cursor_trans[pipe] = temp;
2998
2d41c0b5
PB		 2999 r->wm_linetime[pipe] = p_wm->linetime;
3000}
3001
16160e3d
DL		 3002static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg,
3003 const struct skl_ddb_entry *entry)
3004{
3005 if (entry->end)
3006 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
3007 else
3008 I915_WRITE(reg, 0);
3009}
3010
2d41c0b5
PB		 3011static void skl_write_wm_values(struct drm_i915_private *dev_priv,
3012 const struct skl_wm_values *new)
3013{
3014 struct drm_device *dev = dev_priv->dev;
3015 struct intel_crtc *crtc;
3016
3017 list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
3018 int i, level, max_level = ilk_wm_max_level(dev);
3019 enum pipe pipe = crtc->pipe;
3020
5d374d96
DL		 3021 if (!new->dirty[pipe])
3022 continue;
8211bd5b 3023
5d374d96 3024 I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
8211bd5b 3025
5d374d96
DL		 3026 for (level = 0; level <= max_level; level++) {
3027 for (i = 0; i < intel_num_planes(crtc); i++)
3028 I915_WRITE(PLANE_WM(pipe, i, level),
3029 new->plane[pipe][i][level]);
3030 I915_WRITE(CUR_WM(pipe, level),
3031 new->cursor[pipe][level]);
2d41c0b5 3032 }
5d374d96
DL		 3033 for (i = 0; i < intel_num_planes(crtc); i++)
3034 I915_WRITE(PLANE_WM_TRANS(pipe, i),
3035 new->plane_trans[pipe][i]);
3036 I915_WRITE(CUR_WM_TRANS(pipe), new->cursor_trans[pipe]);
3037
3038 for (i = 0; i < intel_num_planes(crtc); i++)
3039 skl_ddb_entry_write(dev_priv,
3040 PLANE_BUF_CFG(pipe, i),
3041 &new->ddb.plane[pipe][i]);
3042
3043 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
3044 &new->ddb.cursor[pipe]);
2d41c0b5 3045 }
2d41c0b5
PB		 3046}
3047
0e8fb7ba
DL		 3048/*
3049 * When setting up a new DDB allocation arrangement, we need to correctly
3050 * sequence the times at which the new allocations for the pipes are taken into
3051 * account or we'll have pipes fetching from space previously allocated to
3052 * another pipe.
3053 *
3054 * Roughly the sequence looks like:
3055 * 1. re-allocate the pipe(s) with the allocation being reduced and not
3056 * overlapping with a previous light-up pipe (another way to put it is:
3057 * pipes with their new allocation strickly included into their old ones).
3058 * 2. re-allocate the other pipes that get their allocation reduced
3059 * 3. allocate the pipes having their allocation increased
3060 *
3061 * Steps 1. and 2. are here to take care of the following case:
3062 * - Initially DDB looks like this:
3063 * | B | C |
3064 * - enable pipe A.
3065 * - pipe B has a reduced DDB allocation that overlaps with the old pipe C
3066 * allocation
3067 * | A | B | C |
3068 *
3069 * We need to sequence the re-allocation: C, B, A (and not B, C, A).
3070 */
3071
d21b795c
DL		 3072static void
3073skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
0e8fb7ba 3074{
0e8fb7ba
DL		 3075 int plane;
3076
d21b795c
DL		 3077 DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
3078
dd740780 3079 for_each_plane(dev_priv, pipe, plane) {
0e8fb7ba
DL		 3080 I915_WRITE(PLANE_SURF(pipe, plane),
3081 I915_READ(PLANE_SURF(pipe, plane)));
3082 }
3083 I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
3084}
3085
3086static bool
3087skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
3088 const struct skl_ddb_allocation *new,
3089 enum pipe pipe)
3090{
3091 uint16_t old_size, new_size;
3092
3093 old_size = skl_ddb_entry_size(&old->pipe[pipe]);
3094 new_size = skl_ddb_entry_size(&new->pipe[pipe]);
3095
3096 return old_size != new_size &&
3097 new->pipe[pipe].start >= old->pipe[pipe].start &&
3098 new->pipe[pipe].end <= old->pipe[pipe].end;
3099}
3100
3101static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
3102 struct skl_wm_values *new_values)
3103{
3104 struct drm_device *dev = dev_priv->dev;
3105 struct skl_ddb_allocation *cur_ddb, *new_ddb;
3106 bool reallocated[I915_MAX_PIPES] = {false, false, false};
3107 struct intel_crtc *crtc;
3108 enum pipe pipe;
3109
3110 new_ddb = &new_values->ddb;
3111 cur_ddb = &dev_priv->wm.skl_hw.ddb;
3112
3113 /*
3114 * First pass: flush the pipes with the new allocation contained into
3115 * the old space.
3116 *
3117 * We'll wait for the vblank on those pipes to ensure we can safely
3118 * re-allocate the freed space without this pipe fetching from it.
3119 */
3120 for_each_intel_crtc(dev, crtc) {
3121 if (!crtc->active)
3122 continue;
3123
3124 pipe = crtc->pipe;
3125
3126 if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
3127 continue;
3128
d21b795c 3129 skl_wm_flush_pipe(dev_priv, pipe, 1);
0e8fb7ba
DL		 3130 intel_wait_for_vblank(dev, pipe);
3131
3132 reallocated[pipe] = true;
3133 }
3134
3135
3136 /*
3137 * Second pass: flush the pipes that are having their allocation
3138 * reduced, but overlapping with a previous allocation.
3139 *
3140 * Here as well we need to wait for the vblank to make sure the freed
3141 * space is not used anymore.
3142 */
3143 for_each_intel_crtc(dev, crtc) {
3144 if (!crtc->active)
3145 continue;
3146
3147 pipe = crtc->pipe;
3148
3149 if (reallocated[pipe])
3150 continue;
3151
3152 if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
3153 skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
d21b795c 3154 skl_wm_flush_pipe(dev_priv, pipe, 2);
0e8fb7ba 3155 intel_wait_for_vblank(dev, pipe);
d9d8e6b3 3156 reallocated[pipe] = true;
0e8fb7ba 3157 }
0e8fb7ba
DL		 3158 }
3159
3160 /*
3161 * Third pass: flush the pipes that got more space allocated.
3162 *
3163 * We don't need to actively wait for the update here, next vblank
3164 * will just get more DDB space with the correct WM values.
3165 */
3166 for_each_intel_crtc(dev, crtc) {
3167 if (!crtc->active)
3168 continue;
3169
3170 pipe = crtc->pipe;
3171
3172 /*
3173 * At this point, only the pipes more space than before are
3174 * left to re-allocate.
3175 */
3176 if (reallocated[pipe])
3177 continue;
3178
d21b795c 3179 skl_wm_flush_pipe(dev_priv, pipe, 3);
0e8fb7ba
DL		 3180 }
3181}
3182
2d41c0b5
PB		 3183static bool skl_update_pipe_wm(struct drm_crtc *crtc,
3184 struct skl_pipe_wm_parameters *params,
3185 struct intel_wm_config *config,
3186 struct skl_ddb_allocation *ddb, /* out */
3187 struct skl_pipe_wm *pipe_wm /* out */)
3188{
3189 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3190
3191 skl_compute_wm_pipe_parameters(crtc, params);
b9cec075 3192 skl_allocate_pipe_ddb(crtc, config, params, ddb);
2d41c0b5
PB		 3193 skl_compute_pipe_wm(crtc, ddb, params, pipe_wm);
3194
3195 if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm)))
3196 return false;
3197
3198 intel_crtc->wm.skl_active = *pipe_wm;
3199 return true;
3200}
3201
3202static void skl_update_other_pipe_wm(struct drm_device *dev,
3203 struct drm_crtc *crtc,
3204 struct intel_wm_config *config,
3205 struct skl_wm_values *r)
3206{
3207 struct intel_crtc *intel_crtc;
3208 struct intel_crtc *this_crtc = to_intel_crtc(crtc);
3209
3210 /*
3211 * If the WM update hasn't changed the allocation for this_crtc (the
3212 * crtc we are currently computing the new WM values for), other
3213 * enabled crtcs will keep the same allocation and we don't need to
3214 * recompute anything for them.
3215 */
3216 if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
3217 return;
3218
3219 /*
3220 * Otherwise, because of this_crtc being freshly enabled/disabled, the
3221 * other active pipes need new DDB allocation and WM values.
3222 */
3223 list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
3224 base.head) {
3225 struct skl_pipe_wm_parameters params = {};
3226 struct skl_pipe_wm pipe_wm = {};
3227 bool wm_changed;
3228
3229 if (this_crtc->pipe == intel_crtc->pipe)
3230 continue;
3231
3232 if (!intel_crtc->active)
3233 continue;
3234
3235 wm_changed = skl_update_pipe_wm(&intel_crtc->base,
3236 &params, config,
3237 &r->ddb, &pipe_wm);
3238
3239 /*
3240 * If we end up re-computing the other pipe WM values, it's
3241 * because it was really needed, so we expect the WM values to
3242 * be different.
3243 */
3244 WARN_ON(!wm_changed);
3245
3246 skl_compute_wm_results(dev, &params, &pipe_wm, r, intel_crtc);
3247 r->dirty[intel_crtc->pipe] = true;
3248 }
3249}
3250
3251static void skl_update_wm(struct drm_crtc *crtc)
3252{
3253 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3254 struct drm_device *dev = crtc->dev;
3255 struct drm_i915_private *dev_priv = dev->dev_private;
3256 struct skl_pipe_wm_parameters params = {};
3257 struct skl_wm_values *results = &dev_priv->wm.skl_results;
3258 struct skl_pipe_wm pipe_wm = {};
3259 struct intel_wm_config config = {};
3260
3261 memset(results, 0, sizeof(*results));
3262
3263 skl_compute_wm_global_parameters(dev, &config);
3264
3265 if (!skl_update_pipe_wm(crtc, &params, &config,
3266 &results->ddb, &pipe_wm))
3267 return;
3268
3269 skl_compute_wm_results(dev, &params, &pipe_wm, results, intel_crtc);
3270 results->dirty[intel_crtc->pipe] = true;
3271
3272 skl_update_other_pipe_wm(dev, crtc, &config, results);
3273 skl_write_wm_values(dev_priv, results);
0e8fb7ba 3274 skl_flush_wm_values(dev_priv, results);
53b0deb4
DL		 3275
3276 /* store the new configuration */
3277 dev_priv->wm.skl_hw = *results;
2d41c0b5
PB		 3278}
3279
3280static void
3281skl_update_sprite_wm(struct drm_plane *plane, struct drm_crtc *crtc,
3282 uint32_t sprite_width, uint32_t sprite_height,
3283 int pixel_size, bool enabled, bool scaled)
3284{
3285 struct intel_plane *intel_plane = to_intel_plane(plane);
0fda6568 3286 struct drm_framebuffer *fb = plane->state->fb;
2d41c0b5
PB		 3287
3288 intel_plane->wm.enabled = enabled;
3289 intel_plane->wm.scaled = scaled;
3290 intel_plane->wm.horiz_pixels = sprite_width;
3291 intel_plane->wm.vert_pixels = sprite_height;
3292 intel_plane->wm.bytes_per_pixel = pixel_size;
0fda6568
TU		 3293 intel_plane->wm.tiling = DRM_FORMAT_MOD_NONE;
3294 /*
3295 * Framebuffer can be NULL on plane disable, but it does not
3296 * matter for watermarks if we assume no tiling in that case.
3297 */
3298 if (fb)
3299 intel_plane->wm.tiling = fb->modifier[0];
2d41c0b5
PB		 3300
3301 skl_update_wm(crtc);
3302}
3303
820c1980 3304static void ilk_update_wm(struct drm_crtc *crtc)
801bcfff 3305{
7c4a395f 3306 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
46ba614c 3307 struct drm_device *dev = crtc->dev;
801bcfff 3308 struct drm_i915_private *dev_priv = dev->dev_private;
820c1980
ID		 3309 struct ilk_wm_maximums max;
3310 struct ilk_pipe_wm_parameters params = {};
3311 struct ilk_wm_values results = {};
77c122bc 3312 enum intel_ddb_partitioning partitioning;
7c4a395f 3313 struct intel_pipe_wm pipe_wm = {};
198a1e9b 3314 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
a485bfb8 3315 struct intel_wm_config config = {};
7c4a395f 3316
2a44b76b 3317 ilk_compute_wm_parameters(crtc, &params);
7c4a395f
VS		 3318
3319 intel_compute_pipe_wm(crtc, &params, &pipe_wm);
3320
3321 if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
3322 return;
861f3389 3323
7c4a395f 3324 intel_crtc->wm.active = pipe_wm;
861f3389 3325
2a44b76b
VS		 3326 ilk_compute_wm_config(dev, &config);
3327
34982fe1 3328 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
0ba22e26 3329 ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
a485bfb8
VS		 3330
3331 /* 5/6 split only in single pipe config on IVB+ */
ec98c8d1
VS		 3332 if (INTEL_INFO(dev)->gen >= 7 &&
3333 config.num_pipes_active == 1 && config.sprites_enabled) {
34982fe1 3334 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
0ba22e26 3335 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
0362c781 3336
820c1980 3337 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
861f3389 3338 } else {
198a1e9b 3339 best_lp_wm = &lp_wm_1_2;
861f3389
PZ		 3340 }
3341
198a1e9b 3342 partitioning = (best_lp_wm == &lp_wm_1_2) ?
77c122bc 3343 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
801bcfff 3344
820c1980 3345 ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
609cedef 3346
820c1980 3347 ilk_write_wm_values(dev_priv, &results);
1011d8c4
PZ		 3348}
3349
ed57cb8a
DL		 3350static void
3351ilk_update_sprite_wm(struct drm_plane *plane,
3352 struct drm_crtc *crtc,
3353 uint32_t sprite_width, uint32_t sprite_height,
3354 int pixel_size, bool enabled, bool scaled)
526682e9 3355{
8553c18e 3356 struct drm_device *dev = plane->dev;
adf3d35e 3357 struct intel_plane *intel_plane = to_intel_plane(plane);
526682e9 3358
adf3d35e
VS		 3359 intel_plane->wm.enabled = enabled;
3360 intel_plane->wm.scaled = scaled;
3361 intel_plane->wm.horiz_pixels = sprite_width;
ed57cb8a 3362 intel_plane->wm.vert_pixels = sprite_width;
adf3d35e 3363 intel_plane->wm.bytes_per_pixel = pixel_size;
526682e9 3364
8553c18e
VS		 3365 /*
3366 * IVB workaround: must disable low power watermarks for at least
3367 * one frame before enabling scaling. LP watermarks can be re-enabled
3368 * when scaling is disabled.
3369 *
3370 * WaCxSRDisabledForSpriteScaling:ivb
3371 */
3372 if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
3373 intel_wait_for_vblank(dev, intel_plane->pipe);
3374
820c1980 3375 ilk_update_wm(crtc);
526682e9
PZ		 3376}
3377
3078999f
PB		 3378static void skl_pipe_wm_active_state(uint32_t val,
3379 struct skl_pipe_wm *active,
3380 bool is_transwm,
3381 bool is_cursor,
3382 int i,
3383 int level)
3384{
3385 bool is_enabled = (val & PLANE_WM_EN) != 0;
3386
3387 if (!is_transwm) {
3388 if (!is_cursor) {
3389 active->wm[level].plane_en[i] = is_enabled;
3390 active->wm[level].plane_res_b[i] =
3391 val & PLANE_WM_BLOCKS_MASK;
3392 active->wm[level].plane_res_l[i] =
3393 (val >> PLANE_WM_LINES_SHIFT) &
3394 PLANE_WM_LINES_MASK;
3395 } else {
3396 active->wm[level].cursor_en = is_enabled;
3397 active->wm[level].cursor_res_b =
3398 val & PLANE_WM_BLOCKS_MASK;
3399 active->wm[level].cursor_res_l =
3400 (val >> PLANE_WM_LINES_SHIFT) &
3401 PLANE_WM_LINES_MASK;
3402 }
3403 } else {
3404 if (!is_cursor) {
3405 active->trans_wm.plane_en[i] = is_enabled;
3406 active->trans_wm.plane_res_b[i] =
3407 val & PLANE_WM_BLOCKS_MASK;
3408 active->trans_wm.plane_res_l[i] =
3409 (val >> PLANE_WM_LINES_SHIFT) &
3410 PLANE_WM_LINES_MASK;
3411 } else {
3412 active->trans_wm.cursor_en = is_enabled;
3413 active->trans_wm.cursor_res_b =
3414 val & PLANE_WM_BLOCKS_MASK;
3415 active->trans_wm.cursor_res_l =
3416 (val >> PLANE_WM_LINES_SHIFT) &
3417 PLANE_WM_LINES_MASK;
3418 }
3419 }
3420}
3421
3422static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3423{
3424 struct drm_device *dev = crtc->dev;
3425 struct drm_i915_private *dev_priv = dev->dev_private;
3426 struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
3427 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3428 struct skl_pipe_wm *active = &intel_crtc->wm.skl_active;
3429 enum pipe pipe = intel_crtc->pipe;
3430 int level, i, max_level;
3431 uint32_t temp;
3432
3433 max_level = ilk_wm_max_level(dev);
3434
3435 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3436
3437 for (level = 0; level <= max_level; level++) {
3438 for (i = 0; i < intel_num_planes(intel_crtc); i++)
3439 hw->plane[pipe][i][level] =
3440 I915_READ(PLANE_WM(pipe, i, level));
3441 hw->cursor[pipe][level] = I915_READ(CUR_WM(pipe, level));
3442 }
3443
3444 for (i = 0; i < intel_num_planes(intel_crtc); i++)
3445 hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
3446 hw->cursor_trans[pipe] = I915_READ(CUR_WM_TRANS(pipe));
3447
3ef00284 3448 if (!intel_crtc->active)
3078999f
PB		 3449 return;
3450
3451 hw->dirty[pipe] = true;
3452
3453 active->linetime = hw->wm_linetime[pipe];
3454
3455 for (level = 0; level <= max_level; level++) {
3456 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3457 temp = hw->plane[pipe][i][level];
3458 skl_pipe_wm_active_state(temp, active, false,
3459 false, i, level);
3460 }
3461 temp = hw->cursor[pipe][level];
3462 skl_pipe_wm_active_state(temp, active, false, true, i, level);
3463 }
3464
3465 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3466 temp = hw->plane_trans[pipe][i];
3467 skl_pipe_wm_active_state(temp, active, true, false, i, 0);
3468 }
3469
3470 temp = hw->cursor_trans[pipe];
3471 skl_pipe_wm_active_state(temp, active, true, true, i, 0);
3472}
3473
3474void skl_wm_get_hw_state(struct drm_device *dev)
3475{
a269c583
DL		 3476 struct drm_i915_private *dev_priv = dev->dev_private;
3477 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
3078999f
PB		 3478 struct drm_crtc *crtc;
3479
a269c583 3480 skl_ddb_get_hw_state(dev_priv, ddb);
3078999f
PB		 3481 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
3482 skl_pipe_wm_get_hw_state(crtc);
3483}
3484
243e6a44
VS		 3485static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3486{
3487 struct drm_device *dev = crtc->dev;
3488 struct drm_i915_private *dev_priv = dev->dev_private;
820c1980 3489 struct ilk_wm_values *hw = &dev_priv->wm.hw;
243e6a44
VS		 3490 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3491 struct intel_pipe_wm *active = &intel_crtc->wm.active;
3492 enum pipe pipe = intel_crtc->pipe;
3493 static const unsigned int wm0_pipe_reg[] = {
3494 [PIPE_A] = WM0_PIPEA_ILK,
3495 [PIPE_B] = WM0_PIPEB_ILK,
3496 [PIPE_C] = WM0_PIPEC_IVB,
3497 };
3498
3499 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
a42a5719 3500 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
ce0e0713 3501 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
243e6a44 3502
3ef00284 3503 active->pipe_enabled = intel_crtc->active;
2a44b76b
VS		 3504
3505 if (active->pipe_enabled) {
243e6a44
VS		 3506 u32 tmp = hw->wm_pipe[pipe];
3507
3508 /*
3509 * For active pipes LP0 watermark is marked as
3510 * enabled, and LP1+ watermaks as disabled since
3511 * we can't really reverse compute them in case
3512 * multiple pipes are active.
3513 */
3514 active->wm[0].enable = true;
3515 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
3516 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
3517 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
3518 active->linetime = hw->wm_linetime[pipe];
3519 } else {
3520 int level, max_level = ilk_wm_max_level(dev);
3521
3522 /*
3523 * For inactive pipes, all watermark levels
3524 * should be marked as enabled but zeroed,
3525 * which is what we'd compute them to.
3526 */
3527 for (level = 0; level <= max_level; level++)
3528 active->wm[level].enable = true;
3529 }
3530}
3531
3532void ilk_wm_get_hw_state(struct drm_device *dev)
3533{
3534 struct drm_i915_private *dev_priv = dev->dev_private;
820c1980 3535 struct ilk_wm_values *hw = &dev_priv->wm.hw;
243e6a44
VS		 3536 struct drm_crtc *crtc;
3537
70e1e0ec 3538 for_each_crtc(dev, crtc)
243e6a44
VS		 3539 ilk_pipe_wm_get_hw_state(crtc);
3540
3541 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
3542 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
3543 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3544
3545 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
cfa7698b
VS		 3546 if (INTEL_INFO(dev)->gen >= 7) {
3547 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
3548 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
3549 }
243e6a44 3550
a42a5719 3551 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
ac9545fd
VS		 3552 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
3553 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3554 else if (IS_IVYBRIDGE(dev))
3555 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
3556 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
243e6a44
VS		 3557
3558 hw->enable_fbc_wm =
3559 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3560}
3561
b445e3b0
ED		 3562/**
3563 * intel_update_watermarks - update FIFO watermark values based on current modes
3564 *
3565 * Calculate watermark values for the various WM regs based on current mode
3566 * and plane configuration.
3567 *
3568 * There are several cases to deal with here:
3569 * - normal (i.e. non-self-refresh)
3570 * - self-refresh (SR) mode
3571 * - lines are large relative to FIFO size (buffer can hold up to 2)
3572 * - lines are small relative to FIFO size (buffer can hold more than 2
3573 * lines), so need to account for TLB latency
3574 *
3575 * The normal calculation is:
3576 * watermark = dotclock * bytes per pixel * latency
3577 * where latency is platform & configuration dependent (we assume pessimal
3578 * values here).
3579 *
3580 * The SR calculation is:
3581 * watermark = (trunc(latency/line time)+1) * surface width *
3582 * bytes per pixel
3583 * where
3584 * line time = htotal / dotclock
3585 * surface width = hdisplay for normal plane and 64 for cursor
3586 * and latency is assumed to be high, as above.
3587 *
3588 * The final value programmed to the register should always be rounded up,
3589 * and include an extra 2 entries to account for clock crossings.
3590 *
3591 * We don't use the sprite, so we can ignore that. And on Crestline we have
3592 * to set the non-SR watermarks to 8.
3593 */
46ba614c 3594void intel_update_watermarks(struct drm_crtc *crtc)
b445e3b0 3595{
46ba614c 3596 struct drm_i915_private *dev_priv = crtc->dev->dev_private;
b445e3b0
ED		 3597
3598 if (dev_priv->display.update_wm)
46ba614c 3599 dev_priv->display.update_wm(crtc);
b445e3b0
ED		 3600}
3601
adf3d35e
VS		 3602void intel_update_sprite_watermarks(struct drm_plane *plane,
3603 struct drm_crtc *crtc,
ed57cb8a
DL		 3604 uint32_t sprite_width,
3605 uint32_t sprite_height,
3606 int pixel_size,
39db4a4d 3607 bool enabled, bool scaled)
b445e3b0 3608{
adf3d35e 3609 struct drm_i915_private *dev_priv = plane->dev->dev_private;
b445e3b0
ED		 3610
3611 if (dev_priv->display.update_sprite_wm)
ed57cb8a
DL		 3612 dev_priv->display.update_sprite_wm(plane, crtc,
3613 sprite_width, sprite_height,
39db4a4d 3614 pixel_size, enabled, scaled);
b445e3b0
ED		 3615}
3616
9270388e
DV		 3617/**
3618 * Lock protecting IPS related data structures
9270388e
DV		 3619 */
3620DEFINE_SPINLOCK(mchdev_lock);
3621
3622/* Global for IPS driver to get at the current i915 device. Protected by
3623 * mchdev_lock. */
3624static struct drm_i915_private *i915_mch_dev;
3625
2b4e57bd
ED		 3626bool ironlake_set_drps(struct drm_device *dev, u8 val)
3627{
3628 struct drm_i915_private *dev_priv = dev->dev_private;
3629 u16 rgvswctl;
3630
9270388e
DV		 3631 assert_spin_locked(&mchdev_lock);
3632
2b4e57bd
ED		 3633 rgvswctl = I915_READ16(MEMSWCTL);
3634 if (rgvswctl & MEMCTL_CMD_STS) {
3635 DRM_DEBUG("gpu busy, RCS change rejected\n");
3636 return false; /* still busy with another command */
3637 }
3638
3639 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
3640 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
3641 I915_WRITE16(MEMSWCTL, rgvswctl);
3642 POSTING_READ16(MEMSWCTL);
3643
3644 rgvswctl |= MEMCTL_CMD_STS;
3645 I915_WRITE16(MEMSWCTL, rgvswctl);
3646
3647 return true;
3648}
3649
8090c6b9 3650static void ironlake_enable_drps(struct drm_device *dev)
2b4e57bd
ED		 3651{
3652 struct drm_i915_private *dev_priv = dev->dev_private;
3653 u32 rgvmodectl = I915_READ(MEMMODECTL);
3654 u8 fmax, fmin, fstart, vstart;
3655
9270388e
DV		 3656 spin_lock_irq(&mchdev_lock);
3657
2b4e57bd
ED		 3658 /* Enable temp reporting */
3659 I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
3660 I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
3661
3662 /* 100ms RC evaluation intervals */
3663 I915_WRITE(RCUPEI, 100000);
3664 I915_WRITE(RCDNEI, 100000);
3665
3666 /* Set max/min thresholds to 90ms and 80ms respectively */
3667 I915_WRITE(RCBMAXAVG, 90000);
3668 I915_WRITE(RCBMINAVG, 80000);
3669
3670 I915_WRITE(MEMIHYST, 1);
3671
3672 /* Set up min, max, and cur for interrupt handling */
3673 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
3674 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
3675 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
3676 MEMMODE_FSTART_SHIFT;
3677
3678 vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
3679 PXVFREQ_PX_SHIFT;
3680
20e4d407
DV		 3681 dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
3682 dev_priv->ips.fstart = fstart;
2b4e57bd 3683
20e4d407
DV		 3684 dev_priv->ips.max_delay = fstart;
3685 dev_priv->ips.min_delay = fmin;
3686 dev_priv->ips.cur_delay = fstart;
2b4e57bd
ED		 3687
3688 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
3689 fmax, fmin, fstart);
3690
3691 I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
3692
3693 /*
3694 * Interrupts will be enabled in ironlake_irq_postinstall
3695 */
3696
3697 I915_WRITE(VIDSTART, vstart);
3698 POSTING_READ(VIDSTART);
3699
3700 rgvmodectl |= MEMMODE_SWMODE_EN;
3701 I915_WRITE(MEMMODECTL, rgvmodectl);
3702
9270388e 3703 if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
2b4e57bd 3704 DRM_ERROR("stuck trying to change perf mode\n");
9270388e 3705 mdelay(1);
2b4e57bd
ED		 3706
3707 ironlake_set_drps(dev, fstart);
3708
20e4d407 3709 dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
2b4e57bd 3710 I915_READ(0x112e0);
20e4d407
DV		 3711 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
3712 dev_priv->ips.last_count2 = I915_READ(0x112f4);
5ed0bdf2 3713 dev_priv->ips.last_time2 = ktime_get_raw_ns();
9270388e
DV		 3714
3715 spin_unlock_irq(&mchdev_lock);
2b4e57bd
ED		 3716}
3717
8090c6b9 3718static void ironlake_disable_drps(struct drm_device *dev)
2b4e57bd
ED		 3719{
3720 struct drm_i915_private *dev_priv = dev->dev_private;
9270388e
DV		 3721 u16 rgvswctl;
3722
3723 spin_lock_irq(&mchdev_lock);
3724
3725 rgvswctl = I915_READ16(MEMSWCTL);
2b4e57bd
ED		 3726
3727 /* Ack interrupts, disable EFC interrupt */
3728 I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
3729 I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
3730 I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
3731 I915_WRITE(DEIIR, DE_PCU_EVENT);
3732 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
3733
3734 /* Go back to the starting frequency */
20e4d407 3735 ironlake_set_drps(dev, dev_priv->ips.fstart);
9270388e 3736 mdelay(1);
2b4e57bd
ED		 3737 rgvswctl |= MEMCTL_CMD_STS;
3738 I915_WRITE(MEMSWCTL, rgvswctl);
9270388e 3739 mdelay(1);
2b4e57bd 3740
9270388e 3741 spin_unlock_irq(&mchdev_lock);
2b4e57bd
ED		 3742}
3743
acbe9475
DV		 3744/* There's a funny hw issue where the hw returns all 0 when reading from
3745 * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
3746 * ourselves, instead of doing a rmw cycle (which might result in us clearing
3747 * all limits and the gpu stuck at whatever frequency it is at atm).
3748 */
6917c7b9 3749static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val)
2b4e57bd 3750{
7b9e0ae6 3751 u32 limits;
2b4e57bd 3752
20b46e59
DV		 3753 /* Only set the down limit when we've reached the lowest level to avoid
3754 * getting more interrupts, otherwise leave this clear. This prevents a
3755 * race in the hw when coming out of rc6: There's a tiny window where
3756 * the hw runs at the minimal clock before selecting the desired
3757 * frequency, if the down threshold expires in that window we will not
3758 * receive a down interrupt. */
b39fb297
BW		 3759 limits = dev_priv->rps.max_freq_softlimit << 24;
3760 if (val <= dev_priv->rps.min_freq_softlimit)
3761 limits |= dev_priv->rps.min_freq_softlimit << 16;
20b46e59
DV		 3762
3763 return limits;
3764}
3765
dd75fdc8
CW		 3766static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
3767{
3768 int new_power;
3769
3770 new_power = dev_priv->rps.power;
3771 switch (dev_priv->rps.power) {
3772 case LOW_POWER:
b39fb297 3773 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
dd75fdc8
CW		 3774 new_power = BETWEEN;
3775 break;
3776
3777 case BETWEEN:
b39fb297 3778 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
dd75fdc8 3779 new_power = LOW_POWER;
b39fb297 3780 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
dd75fdc8
CW		 3781 new_power = HIGH_POWER;
3782 break;
3783
3784 case HIGH_POWER:
b39fb297 3785 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
dd75fdc8
CW		 3786 new_power = BETWEEN;
3787 break;
3788 }
3789 /* Max/min bins are special */
b39fb297 3790 if (val == dev_priv->rps.min_freq_softlimit)
dd75fdc8 3791 new_power = LOW_POWER;
b39fb297 3792 if (val == dev_priv->rps.max_freq_softlimit)
dd75fdc8
CW		 3793 new_power = HIGH_POWER;
3794 if (new_power == dev_priv->rps.power)
3795 return;
3796
3797 /* Note the units here are not exactly 1us, but 1280ns. */
3798 switch (new_power) {
3799 case LOW_POWER:
3800 /* Upclock if more than 95% busy over 16ms */
3801 I915_WRITE(GEN6_RP_UP_EI, 12500);
3802 I915_WRITE(GEN6_RP_UP_THRESHOLD, 11800);
3803
3804 /* Downclock if less than 85% busy over 32ms */
3805 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3806 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 21250);
3807
3808 I915_WRITE(GEN6_RP_CONTROL,
3809 GEN6_RP_MEDIA_TURBO |
3810 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3811 GEN6_RP_MEDIA_IS_GFX |
3812 GEN6_RP_ENABLE |
3813 GEN6_RP_UP_BUSY_AVG |
3814 GEN6_RP_DOWN_IDLE_AVG);
3815 break;
3816
3817 case BETWEEN:
3818 /* Upclock if more than 90% busy over 13ms */
3819 I915_WRITE(GEN6_RP_UP_EI, 10250);
3820 I915_WRITE(GEN6_RP_UP_THRESHOLD, 9225);
3821
3822 /* Downclock if less than 75% busy over 32ms */
3823 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3824 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 18750);
3825
3826 I915_WRITE(GEN6_RP_CONTROL,
3827 GEN6_RP_MEDIA_TURBO |
3828 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3829 GEN6_RP_MEDIA_IS_GFX |
3830 GEN6_RP_ENABLE |
3831 GEN6_RP_UP_BUSY_AVG |
3832 GEN6_RP_DOWN_IDLE_AVG);
3833 break;
3834
3835 case HIGH_POWER:
3836 /* Upclock if more than 85% busy over 10ms */
3837 I915_WRITE(GEN6_RP_UP_EI, 8000);
3838 I915_WRITE(GEN6_RP_UP_THRESHOLD, 6800);
3839
3840 /* Downclock if less than 60% busy over 32ms */
3841 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3842 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 15000);
3843
3844 I915_WRITE(GEN6_RP_CONTROL,
3845 GEN6_RP_MEDIA_TURBO |
3846 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3847 GEN6_RP_MEDIA_IS_GFX |
3848 GEN6_RP_ENABLE |
3849 GEN6_RP_UP_BUSY_AVG |
3850 GEN6_RP_DOWN_IDLE_AVG);
3851 break;
3852 }
3853
3854 dev_priv->rps.power = new_power;
3855 dev_priv->rps.last_adj = 0;
3856}
3857
2876ce73
CW		 3858static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
3859{
3860 u32 mask = 0;
3861
3862 if (val > dev_priv->rps.min_freq_softlimit)
3863 mask |= GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
3864 if (val < dev_priv->rps.max_freq_softlimit)
3865 mask |= GEN6_PM_RP_UP_THRESHOLD;
3866
7b3c29f6
CW		 3867 mask |= dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED);
3868 mask &= dev_priv->pm_rps_events;
3869
59d02a1f 3870 return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
2876ce73
CW		 3871}
3872
b8a5ff8d
JM		 3873/* gen6_set_rps is called to update the frequency request, but should also be
3874 * called when the range (min_delay and max_delay) is modified so that we can
3875 * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
ffe02b40 3876static void gen6_set_rps(struct drm_device *dev, u8 val)
20b46e59
DV		 3877{
3878 struct drm_i915_private *dev_priv = dev->dev_private;
7b9e0ae6 3879
4fc688ce 3880 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
b39fb297
BW		 3881 WARN_ON(val > dev_priv->rps.max_freq_softlimit);
3882 WARN_ON(val < dev_priv->rps.min_freq_softlimit);
004777cb 3883
eb64cad1
CW		 3884 /* min/max delay may still have been modified so be sure to
3885 * write the limits value.
3886 */
3887 if (val != dev_priv->rps.cur_freq) {
3888 gen6_set_rps_thresholds(dev_priv, val);
b8a5ff8d 3889
50e6a2a7 3890 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
eb64cad1
CW		 3891 I915_WRITE(GEN6_RPNSWREQ,
3892 HSW_FREQUENCY(val));
3893 else
3894 I915_WRITE(GEN6_RPNSWREQ,
3895 GEN6_FREQUENCY(val) |
3896 GEN6_OFFSET(0) |
3897 GEN6_AGGRESSIVE_TURBO);
b8a5ff8d 3898 }
7b9e0ae6 3899
7b9e0ae6
CW		 3900 /* Make sure we continue to get interrupts
3901 * until we hit the minimum or maximum frequencies.
3902 */
eb64cad1 3903 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, gen6_rps_limits(dev_priv, val));
2876ce73 3904 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
7b9e0ae6 3905
d5570a72
BW		 3906 POSTING_READ(GEN6_RPNSWREQ);
3907
b39fb297 3908 dev_priv->rps.cur_freq = val;
be2cde9a 3909 trace_intel_gpu_freq_change(val * 50);
2b4e57bd
ED		 3910}
3911
ffe02b40
VS		 3912static void valleyview_set_rps(struct drm_device *dev, u8 val)
3913{
3914 struct drm_i915_private *dev_priv = dev->dev_private;
3915
3916 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3917 WARN_ON(val > dev_priv->rps.max_freq_softlimit);
3918 WARN_ON(val < dev_priv->rps.min_freq_softlimit);
3919
3920 if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
3921 "Odd GPU freq value\n"))
3922 val &= ~1;
3923
3924 if (val != dev_priv->rps.cur_freq)
3925 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
3926
3927 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3928
3929 dev_priv->rps.cur_freq = val;
3930 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
3931}
3932
76c3552f
D		 3933/* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down
3934 *
3935 * * If Gfx is Idle, then
3936 * 1. Mask Turbo interrupts
3937 * 2. Bring up Gfx clock
3938 * 3. Change the freq to Rpn and wait till P-Unit updates freq
3939 * 4. Clear the Force GFX CLK ON bit so that Gfx can down
3940 * 5. Unmask Turbo interrupts
3941*/
3942static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
3943{
5549d25f
D		 3944 struct drm_device *dev = dev_priv->dev;
3945
21a11fff
VS		 3946 /* CHV and latest VLV don't need to force the gfx clock */
3947 if (IS_CHERRYVIEW(dev) || dev->pdev->revision >= 0xd) {
5549d25f
D		 3948 valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
3949 return;
3950 }
3951
76c3552f
D		 3952 /*
3953 * When we are idle. Drop to min voltage state.
3954 */
3955
b39fb297 3956 if (dev_priv->rps.cur_freq <= dev_priv->rps.min_freq_softlimit)
76c3552f
D		 3957 return;
3958
3959 /* Mask turbo interrupt so that they will not come in between */
f24eeb19
ID		 3960 I915_WRITE(GEN6_PMINTRMSK,
3961 gen6_sanitize_rps_pm_mask(dev_priv, ~0));
76c3552f 3962
650ad970 3963 vlv_force_gfx_clock(dev_priv, true);
76c3552f 3964
b39fb297 3965 dev_priv->rps.cur_freq = dev_priv->rps.min_freq_softlimit;
76c3552f
D		 3966
3967 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ,
b39fb297 3968 dev_priv->rps.min_freq_softlimit);
76c3552f
D		 3969
3970 if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS))
2837ac40 3971 & GENFREQSTATUS) == 0, 100))
76c3552f
D		 3972 DRM_ERROR("timed out waiting for Punit\n");
3973
650ad970 3974 vlv_force_gfx_clock(dev_priv, false);
76c3552f 3975
2876ce73
CW		 3976 I915_WRITE(GEN6_PMINTRMSK,
3977 gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
76c3552f
D		 3978}
3979
b29c19b6
CW		 3980void gen6_rps_idle(struct drm_i915_private *dev_priv)
3981{
691bb717
DL		 3982 struct drm_device *dev = dev_priv->dev;
3983
b29c19b6 3984 mutex_lock(&dev_priv->rps.hw_lock);
c0951f0c 3985 if (dev_priv->rps.enabled) {
21a11fff 3986 if (IS_VALLEYVIEW(dev))
76c3552f 3987 vlv_set_rps_idle(dev_priv);
7526ed79 3988 else
b39fb297 3989 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
c0951f0c
CW		 3990 dev_priv->rps.last_adj = 0;
3991 }
b29c19b6
CW		 3992 mutex_unlock(&dev_priv->rps.hw_lock);
3993}
3994
3995void gen6_rps_boost(struct drm_i915_private *dev_priv)
3996{
3997 mutex_lock(&dev_priv->rps.hw_lock);
c0951f0c 3998 if (dev_priv->rps.enabled) {
ffe02b40 3999 intel_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
c0951f0c
CW		 4000 dev_priv->rps.last_adj = 0;
4001 }
b29c19b6
CW		 4002 mutex_unlock(&dev_priv->rps.hw_lock);
4003}
4004
ffe02b40 4005void intel_set_rps(struct drm_device *dev, u8 val)
0a073b84 4006{
ffe02b40
VS		 4007 if (IS_VALLEYVIEW(dev))
4008 valleyview_set_rps(dev, val);
4009 else
4010 gen6_set_rps(dev, val);
0a073b84
JB		 4011}
4012
20e49366
ZW		 4013static void gen9_disable_rps(struct drm_device *dev)
4014{
4015 struct drm_i915_private *dev_priv = dev->dev_private;
4016
4017 I915_WRITE(GEN6_RC_CONTROL, 0);
38c23527 4018 I915_WRITE(GEN9_PG_ENABLE, 0);
20e49366
ZW		 4019}
4020
44fc7d5c 4021static void gen6_disable_rps(struct drm_device *dev)
d20d4f0c
JB		 4022{
4023 struct drm_i915_private *dev_priv = dev->dev_private;
4024
4025 I915_WRITE(GEN6_RC_CONTROL, 0);
44fc7d5c 4026 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
44fc7d5c
DV		 4027}
4028
38807746
D		 4029static void cherryview_disable_rps(struct drm_device *dev)
4030{
4031 struct drm_i915_private *dev_priv = dev->dev_private;
4032
4033 I915_WRITE(GEN6_RC_CONTROL, 0);
4034}
4035
44fc7d5c
DV		 4036static void valleyview_disable_rps(struct drm_device *dev)
4037{
4038 struct drm_i915_private *dev_priv = dev->dev_private;
4039
98a2e5f9
D		 4040 /* we're doing forcewake before Disabling RC6,
4041 * This what the BIOS expects when going into suspend */
59bad947 4042 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
98a2e5f9 4043
44fc7d5c 4044 I915_WRITE(GEN6_RC_CONTROL, 0);
d20d4f0c 4045
59bad947 4046 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
d20d4f0c
JB		 4047}
4048
dc39fff7
BW		 4049static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
4050{
91ca689a
ID		 4051 if (IS_VALLEYVIEW(dev)) {
4052 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1)))
4053 mode = GEN6_RC_CTL_RC6_ENABLE;
4054 else
4055 mode = 0;
4056 }
58abf1da
RV		 4057 if (HAS_RC6p(dev))
4058 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
4059 (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
4060 (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
4061 (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
4062
4063 else
4064 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
4065 (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
dc39fff7
BW		 4066}
4067
e6069ca8 4068static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
2b4e57bd 4069{
eb4926e4
DL		 4070 /* No RC6 before Ironlake */
4071 if (INTEL_INFO(dev)->gen < 5)
4072 return 0;
4073
e6069ca8
ID		 4074 /* RC6 is only on Ironlake mobile not on desktop */
4075 if (INTEL_INFO(dev)->gen == 5 && !IS_IRONLAKE_M(dev))
4076 return 0;
4077
456470eb 4078 /* Respect the kernel parameter if it is set */
e6069ca8
ID		 4079 if (enable_rc6 >= 0) {
4080 int mask;
4081
58abf1da 4082 if (HAS_RC6p(dev))
e6069ca8
ID		 4083 mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE |
4084 INTEL_RC6pp_ENABLE;
4085 else
4086 mask = INTEL_RC6_ENABLE;
4087
4088 if ((enable_rc6 & mask) != enable_rc6)
8dfd1f04
DV		 4089 DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
4090 enable_rc6 & mask, enable_rc6, mask);
e6069ca8
ID		 4091
4092 return enable_rc6 & mask;
4093 }
2b4e57bd 4094
6567d748
CW		 4095 /* Disable RC6 on Ironlake */
4096 if (INTEL_INFO(dev)->gen == 5)
4097 return 0;
2b4e57bd 4098
8bade1ad 4099 if (IS_IVYBRIDGE(dev))
cca84a1f 4100 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
8bade1ad
BW		 4101
4102 return INTEL_RC6_ENABLE;
2b4e57bd
ED		 4103}
4104
e6069ca8
ID		 4105int intel_enable_rc6(const struct drm_device *dev)
4106{
4107 return i915.enable_rc6;
4108}
4109
93ee2920 4110static void gen6_init_rps_frequencies(struct drm_device *dev)
3280e8b0 4111{
93ee2920
TR		 4112 struct drm_i915_private *dev_priv = dev->dev_private;
4113 uint32_t rp_state_cap;
4114 u32 ddcc_status = 0;
4115 int ret;
4116
4117 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
3280e8b0
BW		 4118 /* All of these values are in units of 50MHz */
4119 dev_priv->rps.cur_freq = 0;
93ee2920 4120 /* static values from HW: RP0 > RP1 > RPn (min_freq) */
3280e8b0 4121 dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
93ee2920 4122 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
3280e8b0 4123 dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
3280e8b0
BW		 4124 /* hw_max = RP0 until we check for overclocking */
4125 dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
4126
93ee2920
TR		 4127 dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
4128 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
4129 ret = sandybridge_pcode_read(dev_priv,
4130 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
4131 &ddcc_status);
4132 if (0 == ret)
4133 dev_priv->rps.efficient_freq =
46efa4ab
TR		 4134 clamp_t(u8,
4135 ((ddcc_status >> 8) & 0xff),
4136 dev_priv->rps.min_freq,
4137 dev_priv->rps.max_freq);
93ee2920
TR		 4138 }
4139
3280e8b0
BW		 4140 /* Preserve min/max settings in case of re-init */
4141 if (dev_priv->rps.max_freq_softlimit == 0)
4142 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4143
93ee2920
TR		 4144 if (dev_priv->rps.min_freq_softlimit == 0) {
4145 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
4146 dev_priv->rps.min_freq_softlimit =
f4ab408c
TR		 4147 /* max(RPe, 450 MHz) */
4148 max(dev_priv->rps.efficient_freq, (u8) 9);
93ee2920
TR		 4149 else
4150 dev_priv->rps.min_freq_softlimit =
4151 dev_priv->rps.min_freq;
4152 }
3280e8b0
BW		 4153}
4154
b6fef0ef 4155/* See the Gen9_GT_PM_Programming_Guide doc for the below */
20e49366 4156static void gen9_enable_rps(struct drm_device *dev)
b6fef0ef
JB		 4157{
4158 struct drm_i915_private *dev_priv = dev->dev_private;
4159
4160 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4161
ba1c554c
DL		 4162 gen6_init_rps_frequencies(dev);
4163
b6fef0ef
JB		 4164 I915_WRITE(GEN6_RPNSWREQ, 0xc800000);
4165 I915_WRITE(GEN6_RC_VIDEO_FREQ, 0xc800000);
4166
4167 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 0xf4240);
4168 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, 0x12060000);
4169 I915_WRITE(GEN6_RP_UP_THRESHOLD, 0xe808);
4170 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 0x3bd08);
4171 I915_WRITE(GEN6_RP_UP_EI, 0x101d0);
4172 I915_WRITE(GEN6_RP_DOWN_EI, 0x55730);
4173 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
4174 I915_WRITE(GEN6_PMINTRMSK, 0x6);
4175 I915_WRITE(GEN6_RP_CONTROL, GEN6_RP_MEDIA_TURBO |
4176 GEN6_RP_MEDIA_HW_MODE | GEN6_RP_MEDIA_IS_GFX |
4177 GEN6_RP_ENABLE | GEN6_RP_UP_BUSY_AVG |
4178 GEN6_RP_DOWN_IDLE_AVG);
4179
4180 gen6_enable_rps_interrupts(dev);
4181
4182 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4183}
4184
4185static void gen9_enable_rc6(struct drm_device *dev)
20e49366
ZW		 4186{
4187 struct drm_i915_private *dev_priv = dev->dev_private;
4188 struct intel_engine_cs *ring;
4189 uint32_t rc6_mask = 0;
4190 int unused;
4191
4192 /* 1a: Software RC state - RC0 */
4193 I915_WRITE(GEN6_RC_STATE, 0);
4194
4195 /* 1b: Get forcewake during program sequence. Although the driver
4196 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
59bad947 4197 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
20e49366
ZW		 4198
4199 /* 2a: Disable RC states. */
4200 I915_WRITE(GEN6_RC_CONTROL, 0);
4201
4202 /* 2b: Program RC6 thresholds.*/
4203 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
4204 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4205 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4206 for_each_ring(ring, dev_priv, unused)
4207 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4208 I915_WRITE(GEN6_RC_SLEEP, 0);
4209 I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
4210
38c23527
ZW		 4211 /* 2c: Program Coarse Power Gating Policies. */
4212 I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
4213 I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
4214
20e49366
ZW		 4215 /* 3a: Enable RC6 */
4216 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4217 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
4218 DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
4219 "on" : "off");
4220 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4221 GEN6_RC_CTL_EI_MODE(1) |
4222 rc6_mask);
4223
38c23527
ZW		 4224 /* 3b: Enable Coarse Power Gating only when RC6 is enabled */
4225 I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? 3 : 0);
4226
59bad947 4227 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
20e49366
ZW		 4228
4229}
4230
6edee7f3
BW		 4231static void gen8_enable_rps(struct drm_device *dev)
4232{
4233 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 4234 struct intel_engine_cs *ring;
93ee2920 4235 uint32_t rc6_mask = 0;
6edee7f3
BW		 4236 int unused;
4237
4238 /* 1a: Software RC state - RC0 */
4239 I915_WRITE(GEN6_RC_STATE, 0);
4240
4241 /* 1c & 1d: Get forcewake during program sequence. Although the driver
4242 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
59bad947 4243 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6edee7f3
BW		 4244
4245 /* 2a: Disable RC states. */
4246 I915_WRITE(GEN6_RC_CONTROL, 0);
4247
93ee2920
TR		 4248 /* Initialize rps frequencies */
4249 gen6_init_rps_frequencies(dev);
6edee7f3
BW		 4250
4251 /* 2b: Program RC6 thresholds.*/
4252 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
4253 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4254 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4255 for_each_ring(ring, dev_priv, unused)
4256 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4257 I915_WRITE(GEN6_RC_SLEEP, 0);
0d68b25e
TR		 4258 if (IS_BROADWELL(dev))
4259 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
4260 else
4261 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
6edee7f3
BW		 4262
4263 /* 3: Enable RC6 */
4264 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4265 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
abbf9d2c 4266 intel_print_rc6_info(dev, rc6_mask);
0d68b25e
TR		 4267 if (IS_BROADWELL(dev))
4268 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4269 GEN7_RC_CTL_TO_MODE |
4270 rc6_mask);
4271 else
4272 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4273 GEN6_RC_CTL_EI_MODE(1) |
4274 rc6_mask);
6edee7f3
BW		 4275
4276 /* 4 Program defaults and thresholds for RPS*/
f9bdc585
BW		 4277 I915_WRITE(GEN6_RPNSWREQ,
4278 HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4279 I915_WRITE(GEN6_RC_VIDEO_FREQ,
4280 HSW_FREQUENCY(dev_priv->rps.rp1_freq));
7526ed79
DV		 4281 /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
4282 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
4283
4284 /* Docs recommend 900MHz, and 300 MHz respectively */
4285 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
4286 dev_priv->rps.max_freq_softlimit << 24 |
4287 dev_priv->rps.min_freq_softlimit << 16);
4288
4289 I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
4290 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
4291 I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
4292 I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
4293
4294 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
6edee7f3
BW		 4295
4296 /* 5: Enable RPS */
7526ed79
DV		 4297 I915_WRITE(GEN6_RP_CONTROL,
4298 GEN6_RP_MEDIA_TURBO |
4299 GEN6_RP_MEDIA_HW_NORMAL_MODE |
4300 GEN6_RP_MEDIA_IS_GFX |
4301 GEN6_RP_ENABLE |
4302 GEN6_RP_UP_BUSY_AVG |
4303 GEN6_RP_DOWN_IDLE_AVG);
4304
4305 /* 6: Ring frequency + overclocking (our driver does this later */
4306
c7f3153a
TR		 4307 dev_priv->rps.power = HIGH_POWER; /* force a reset */
4308 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
7526ed79 4309
59bad947 4310 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6edee7f3
BW		 4311}
4312
79f5b2c7 4313static void gen6_enable_rps(struct drm_device *dev)
2b4e57bd 4314{
79f5b2c7 4315 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 4316 struct intel_engine_cs *ring;
d060c169 4317 u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
2b4e57bd 4318 u32 gtfifodbg;
2b4e57bd 4319 int rc6_mode;
42c0526c 4320 int i, ret;
2b4e57bd 4321
4fc688ce 4322 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
79f5b2c7 4323
2b4e57bd
ED		 4324 /* Here begins a magic sequence of register writes to enable
4325 * auto-downclocking.
4326 *
4327 * Perhaps there might be some value in exposing these to
4328 * userspace...
4329 */
4330 I915_WRITE(GEN6_RC_STATE, 0);
2b4e57bd
ED		 4331
4332 /* Clear the DBG now so we don't confuse earlier errors */
4333 if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4334 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
4335 I915_WRITE(GTFIFODBG, gtfifodbg);
4336 }
4337
59bad947 4338 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
2b4e57bd 4339
93ee2920
TR		 4340 /* Initialize rps frequencies */
4341 gen6_init_rps_frequencies(dev);
dd0a1aa1 4342
2b4e57bd
ED		 4343 /* disable the counters and set deterministic thresholds */
4344 I915_WRITE(GEN6_RC_CONTROL, 0);
4345
4346 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
4347 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
4348 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
4349 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
4350 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
4351
b4519513
CW		 4352 for_each_ring(ring, dev_priv, i)
4353 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
2b4e57bd
ED		 4354
4355 I915_WRITE(GEN6_RC_SLEEP, 0);
4356 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
29c78f60 4357 if (IS_IVYBRIDGE(dev))
351aa566
SM		 4358 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
4359 else
4360 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
0920a487 4361 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
2b4e57bd
ED		 4362 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
4363
5a7dc92a 4364 /* Check if we are enabling RC6 */
2b4e57bd
ED		 4365 rc6_mode = intel_enable_rc6(dev_priv->dev);
4366 if (rc6_mode & INTEL_RC6_ENABLE)
4367 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
4368
5a7dc92a
ED		 4369 /* We don't use those on Haswell */
4370 if (!IS_HASWELL(dev)) {
4371 if (rc6_mode & INTEL_RC6p_ENABLE)
4372 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
2b4e57bd 4373
5a7dc92a
ED		 4374 if (rc6_mode & INTEL_RC6pp_ENABLE)
4375 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
4376 }
2b4e57bd 4377
dc39fff7 4378 intel_print_rc6_info(dev, rc6_mask);
2b4e57bd
ED		 4379
4380 I915_WRITE(GEN6_RC_CONTROL,
4381 rc6_mask |
4382 GEN6_RC_CTL_EI_MODE(1) |
4383 GEN6_RC_CTL_HW_ENABLE);
4384
dd75fdc8
CW		 4385 /* Power down if completely idle for over 50ms */
4386 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
2b4e57bd 4387 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
2b4e57bd 4388
42c0526c 4389 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
d060c169 4390 if (ret)
42c0526c 4391 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
d060c169
BW		 4392
4393 ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
4394 if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
4395 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
b39fb297 4396 (dev_priv->rps.max_freq_softlimit & 0xff) * 50,
d060c169 4397 (pcu_mbox & 0xff) * 50);
b39fb297 4398 dev_priv->rps.max_freq = pcu_mbox & 0xff;
2b4e57bd
ED		 4399 }
4400
dd75fdc8 4401 dev_priv->rps.power = HIGH_POWER; /* force a reset */
b39fb297 4402 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
2b4e57bd 4403
31643d54
BW		 4404 rc6vids = 0;
4405 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
4406 if (IS_GEN6(dev) && ret) {
4407 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
4408 } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
4409 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
4410 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
4411 rc6vids &= 0xffff00;
4412 rc6vids |= GEN6_ENCODE_RC6_VID(450);
4413 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
4414 if (ret)
4415 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
4416 }
4417
59bad947 4418 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
2b4e57bd
ED		 4419}
4420
c2bc2fc5 4421static void __gen6_update_ring_freq(struct drm_device *dev)
2b4e57bd 4422{
79f5b2c7 4423 struct drm_i915_private *dev_priv = dev->dev_private;
2b4e57bd 4424 int min_freq = 15;
3ebecd07
CW		 4425 unsigned int gpu_freq;
4426 unsigned int max_ia_freq, min_ring_freq;
2b4e57bd 4427 int scaling_factor = 180;
eda79642 4428 struct cpufreq_policy *policy;
2b4e57bd 4429
4fc688ce 4430 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
79f5b2c7 4431
eda79642
BW		 4432 policy = cpufreq_cpu_get(0);
4433 if (policy) {
4434 max_ia_freq = policy->cpuinfo.max_freq;
4435 cpufreq_cpu_put(policy);
4436 } else {
4437 /*
4438 * Default to measured freq if none found, PCU will ensure we
4439 * don't go over
4440 */
2b4e57bd 4441 max_ia_freq = tsc_khz;
eda79642 4442 }
2b4e57bd
ED		 4443
4444 /* Convert from kHz to MHz */
4445 max_ia_freq /= 1000;
4446
153b4b95 4447 min_ring_freq = I915_READ(DCLK) & 0xf;
f6aca45c
BW		 4448 /* convert DDR frequency from units of 266.6MHz to bandwidth */
4449 min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3ebecd07 4450
2b4e57bd
ED		 4451 /*
4452 * For each potential GPU frequency, load a ring frequency we'd like
4453 * to use for memory access. We do this by specifying the IA frequency
4454 * the PCU should use as a reference to determine the ring frequency.
4455 */
6985b352 4456 for (gpu_freq = dev_priv->rps.max_freq; gpu_freq >= dev_priv->rps.min_freq;
2b4e57bd 4457 gpu_freq--) {
6985b352 4458 int diff = dev_priv->rps.max_freq - gpu_freq;
3ebecd07
CW		 4459 unsigned int ia_freq = 0, ring_freq = 0;
4460
46c764d4
BW		 4461 if (INTEL_INFO(dev)->gen >= 8) {
4462 /* max(2 * GT, DDR). NB: GT is 50MHz units */
4463 ring_freq = max(min_ring_freq, gpu_freq);
4464 } else if (IS_HASWELL(dev)) {
f6aca45c 4465 ring_freq = mult_frac(gpu_freq, 5, 4);
3ebecd07
CW		 4466 ring_freq = max(min_ring_freq, ring_freq);
4467 /* leave ia_freq as the default, chosen by cpufreq */
4468 } else {
4469 /* On older processors, there is no separate ring
4470 * clock domain, so in order to boost the bandwidth
4471 * of the ring, we need to upclock the CPU (ia_freq).
4472 *
4473 * For GPU frequencies less than 750MHz,
4474 * just use the lowest ring freq.
4475 */
4476 if (gpu_freq < min_freq)
4477 ia_freq = 800;
4478 else
4479 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
4480 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
4481 }
2b4e57bd 4482
42c0526c
BW		 4483 sandybridge_pcode_write(dev_priv,
4484 GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3ebecd07
CW		 4485 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
4486 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
4487 gpu_freq);
2b4e57bd 4488 }
2b4e57bd
ED		 4489}
4490
c2bc2fc5
ID		 4491void gen6_update_ring_freq(struct drm_device *dev)
4492{
4493 struct drm_i915_private *dev_priv = dev->dev_private;
4494
4495 if (INTEL_INFO(dev)->gen < 6 || IS_VALLEYVIEW(dev))
4496 return;
4497
4498 mutex_lock(&dev_priv->rps.hw_lock);
4499 __gen6_update_ring_freq(dev);
4500 mutex_unlock(&dev_priv->rps.hw_lock);
4501}
4502
03af2045 4503static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
2b6b3a09 4504{
095acd5f 4505 struct drm_device *dev = dev_priv->dev;
2b6b3a09
D		 4506 u32 val, rp0;
4507
095acd5f
D		 4508 if (dev->pdev->revision >= 0x20) {
4509 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
2b6b3a09 4510
095acd5f
D		 4511 switch (INTEL_INFO(dev)->eu_total) {
4512 case 8:
4513 /* (2 * 4) config */
4514 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
4515 break;
4516 case 12:
4517 /* (2 * 6) config */
4518 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
4519 break;
4520 case 16:
4521 /* (2 * 8) config */
4522 default:
4523 /* Setting (2 * 8) Min RP0 for any other combination */
4524 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
4525 break;
4526 }
4527 rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
4528 } else {
4529 /* For pre-production hardware */
4530 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
4531 rp0 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) &
4532 PUNIT_GPU_STATUS_MAX_FREQ_MASK;
4533 }
2b6b3a09
D		 4534 return rp0;
4535}
4536
4537static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
4538{
4539 u32 val, rpe;
4540
4541 val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
4542 rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
4543
4544 return rpe;
4545}
4546
7707df4a
D		 4547static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
4548{
095acd5f 4549 struct drm_device *dev = dev_priv->dev;
7707df4a
D		 4550 u32 val, rp1;
4551
095acd5f
D		 4552 if (dev->pdev->revision >= 0x20) {
4553 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
4554 rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
4555 } else {
4556 /* For pre-production hardware */
4557 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4558 rp1 = ((val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) &
4559 PUNIT_GPU_STATUS_MAX_FREQ_MASK);
4560 }
7707df4a
D		 4561 return rp1;
4562}
4563
03af2045 4564static int cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
2b6b3a09 4565{
095acd5f 4566 struct drm_device *dev = dev_priv->dev;
2b6b3a09
D		 4567 u32 val, rpn;
4568
095acd5f
D		 4569 if (dev->pdev->revision >= 0x20) {
4570 val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
4571 rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
4572 FB_GFX_FREQ_FUSE_MASK);
4573 } else { /* For pre-production hardware */
4574 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
4575 rpn = ((val >> PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT) &
4576 PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK);
4577 }
4578
2b6b3a09
D		 4579 return rpn;
4580}
4581
f8f2b001
D		 4582static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
4583{
4584 u32 val, rp1;
4585
4586 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
4587
4588 rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
4589
4590 return rp1;
4591}
4592
03af2045 4593static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
0a073b84
JB		 4594{
4595 u32 val, rp0;
4596
64936258 4597 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
0a073b84
JB		 4598
4599 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
4600 /* Clamp to max */
4601 rp0 = min_t(u32, rp0, 0xea);
4602
4603 return rp0;
4604}
4605
4606static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
4607{
4608 u32 val, rpe;
4609
64936258 4610 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
0a073b84 4611 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
64936258 4612 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
0a073b84
JB		 4613 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
4614
4615 return rpe;
4616}
4617
03af2045 4618static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
0a073b84 4619{
64936258 4620 return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
0a073b84
JB		 4621}
4622
ae48434c
ID		 4623/* Check that the pctx buffer wasn't move under us. */
4624static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
4625{
4626 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
4627
4628 WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
4629 dev_priv->vlv_pctx->stolen->start);
4630}
4631
38807746
D		 4632
4633/* Check that the pcbr address is not empty. */
4634static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
4635{
4636 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
4637
4638 WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
4639}
4640
4641static void cherryview_setup_pctx(struct drm_device *dev)
4642{
4643 struct drm_i915_private *dev_priv = dev->dev_private;
4644 unsigned long pctx_paddr, paddr;
4645 struct i915_gtt *gtt = &dev_priv->gtt;
4646 u32 pcbr;
4647 int pctx_size = 32*1024;
4648
4649 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
4650
4651 pcbr = I915_READ(VLV_PCBR);
4652 if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
ce611ef8 4653 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
38807746
D		 4654 paddr = (dev_priv->mm.stolen_base +
4655 (gtt->stolen_size - pctx_size));
4656
4657 pctx_paddr = (paddr & (~4095));
4658 I915_WRITE(VLV_PCBR, pctx_paddr);
4659 }
ce611ef8
VS		 4660
4661 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
38807746
D		 4662}
4663
c9cddffc
JB		 4664static void valleyview_setup_pctx(struct drm_device *dev)
4665{
4666 struct drm_i915_private *dev_priv = dev->dev_private;
4667 struct drm_i915_gem_object *pctx;
4668 unsigned long pctx_paddr;
4669 u32 pcbr;
4670 int pctx_size = 24*1024;
4671
17b0c1f7
ID		 4672 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
4673
c9cddffc
JB		 4674 pcbr = I915_READ(VLV_PCBR);
4675 if (pcbr) {
4676 /* BIOS set it up already, grab the pre-alloc'd space */
4677 int pcbr_offset;
4678
4679 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
4680 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
4681 pcbr_offset,
190d6cd5 4682 I915_GTT_OFFSET_NONE,
c9cddffc
JB		 4683 pctx_size);
4684 goto out;
4685 }
4686
ce611ef8
VS		 4687 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
4688
c9cddffc
JB		 4689 /*
4690 * From the Gunit register HAS:
4691 * The Gfx driver is expected to program this register and ensure
4692 * proper allocation within Gfx stolen memory. For example, this
4693 * register should be programmed such than the PCBR range does not
4694 * overlap with other ranges, such as the frame buffer, protected
4695 * memory, or any other relevant ranges.
4696 */
4697 pctx = i915_gem_object_create_stolen(dev, pctx_size);
4698 if (!pctx) {
4699 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
4700 return;
4701 }
4702
4703 pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
4704 I915_WRITE(VLV_PCBR, pctx_paddr);
4705
4706out:
ce611ef8 4707 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
c9cddffc
JB		 4708 dev_priv->vlv_pctx = pctx;
4709}
4710
ae48434c
ID		 4711static void valleyview_cleanup_pctx(struct drm_device *dev)
4712{
4713 struct drm_i915_private *dev_priv = dev->dev_private;
4714
4715 if (WARN_ON(!dev_priv->vlv_pctx))
4716 return;
4717
4718 drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
4719 dev_priv->vlv_pctx = NULL;
4720}
4721
4e80519e
ID		 4722static void valleyview_init_gt_powersave(struct drm_device *dev)
4723{
4724 struct drm_i915_private *dev_priv = dev->dev_private;
2bb25c17 4725 u32 val;
4e80519e
ID		 4726
4727 valleyview_setup_pctx(dev);
4728
4729 mutex_lock(&dev_priv->rps.hw_lock);
4730
2bb25c17
VS		 4731 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4732 switch ((val >> 6) & 3) {
4733 case 0:
4734 case 1:
4735 dev_priv->mem_freq = 800;
4736 break;
4737 case 2:
4738 dev_priv->mem_freq = 1066;
4739 break;
4740 case 3:
4741 dev_priv->mem_freq = 1333;
4742 break;
4743 }
80b83b62 4744 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
2bb25c17 4745
4e80519e
ID		 4746 dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
4747 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
4748 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7c59a9c1 4749 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
4e80519e
ID		 4750 dev_priv->rps.max_freq);
4751
4752 dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
4753 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7c59a9c1 4754 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
4e80519e
ID		 4755 dev_priv->rps.efficient_freq);
4756
f8f2b001
D		 4757 dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
4758 DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
7c59a9c1 4759 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
f8f2b001
D		 4760 dev_priv->rps.rp1_freq);
4761
4e80519e
ID		 4762 dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
4763 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7c59a9c1 4764 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
4e80519e
ID		 4765 dev_priv->rps.min_freq);
4766
4767 /* Preserve min/max settings in case of re-init */
4768 if (dev_priv->rps.max_freq_softlimit == 0)
4769 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4770
4771 if (dev_priv->rps.min_freq_softlimit == 0)
4772 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
4773
4774 mutex_unlock(&dev_priv->rps.hw_lock);
4775}
4776
38807746
D		 4777static void cherryview_init_gt_powersave(struct drm_device *dev)
4778{
2b6b3a09 4779 struct drm_i915_private *dev_priv = dev->dev_private;
2bb25c17 4780 u32 val;
2b6b3a09 4781
38807746 4782 cherryview_setup_pctx(dev);
2b6b3a09
D		 4783
4784 mutex_lock(&dev_priv->rps.hw_lock);
4785
c6e8f39d
VS		 4786 mutex_lock(&dev_priv->dpio_lock);
4787 val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
4788 mutex_unlock(&dev_priv->dpio_lock);
4789
2bb25c17
VS		 4790 switch ((val >> 2) & 0x7) {
4791 case 0:
4792 case 1:
4793 dev_priv->rps.cz_freq = 200;
4794 dev_priv->mem_freq = 1600;
4795 break;
4796 case 2:
4797 dev_priv->rps.cz_freq = 267;
4798 dev_priv->mem_freq = 1600;
4799 break;
4800 case 3:
4801 dev_priv->rps.cz_freq = 333;
4802 dev_priv->mem_freq = 2000;
4803 break;
4804 case 4:
4805 dev_priv->rps.cz_freq = 320;
4806 dev_priv->mem_freq = 1600;
4807 break;
4808 case 5:
4809 dev_priv->rps.cz_freq = 400;
4810 dev_priv->mem_freq = 1600;
4811 break;
4812 }
80b83b62 4813 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
2bb25c17 4814
2b6b3a09
D		 4815 dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
4816 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
4817 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7c59a9c1 4818 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
2b6b3a09
D		 4819 dev_priv->rps.max_freq);
4820
4821 dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
4822 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7c59a9c1 4823 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
2b6b3a09
D		 4824 dev_priv->rps.efficient_freq);
4825
7707df4a
D		 4826 dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
4827 DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
7c59a9c1 4828 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
7707df4a
D		 4829 dev_priv->rps.rp1_freq);
4830
2b6b3a09
D		 4831 dev_priv->rps.min_freq = cherryview_rps_min_freq(dev_priv);
4832 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7c59a9c1 4833 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
2b6b3a09
D		 4834 dev_priv->rps.min_freq);
4835
1c14762d
VS		 4836 WARN_ONCE((dev_priv->rps.max_freq |
4837 dev_priv->rps.efficient_freq |
4838 dev_priv->rps.rp1_freq |
4839 dev_priv->rps.min_freq) & 1,
4840 "Odd GPU freq values\n");
4841
2b6b3a09
D		 4842 /* Preserve min/max settings in case of re-init */
4843 if (dev_priv->rps.max_freq_softlimit == 0)
4844 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4845
4846 if (dev_priv->rps.min_freq_softlimit == 0)
4847 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
4848
4849 mutex_unlock(&dev_priv->rps.hw_lock);
38807746
D		 4850}
4851
4e80519e
ID		 4852static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
4853{
4854 valleyview_cleanup_pctx(dev);
4855}
4856
38807746
D		 4857static void cherryview_enable_rps(struct drm_device *dev)
4858{
4859 struct drm_i915_private *dev_priv = dev->dev_private;
4860 struct intel_engine_cs *ring;
2b6b3a09 4861 u32 gtfifodbg, val, rc6_mode = 0, pcbr;
38807746
D		 4862 int i;
4863
4864 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4865
4866 gtfifodbg = I915_READ(GTFIFODBG);
4867 if (gtfifodbg) {
4868 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
4869 gtfifodbg);
4870 I915_WRITE(GTFIFODBG, gtfifodbg);
4871 }
4872
4873 cherryview_check_pctx(dev_priv);
4874
4875 /* 1a & 1b: Get forcewake during program sequence. Although the driver
4876 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
59bad947 4877 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
38807746 4878
160614a2
VS		 4879 /* Disable RC states. */
4880 I915_WRITE(GEN6_RC_CONTROL, 0);
4881
38807746
D		 4882 /* 2a: Program RC6 thresholds.*/
4883 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
4884 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4885 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4886
4887 for_each_ring(ring, dev_priv, i)
4888 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4889 I915_WRITE(GEN6_RC_SLEEP, 0);
4890
af5a75a3
VS		 4891 /* TO threshold set to 1750 us ( 0x557 * 1.28 us) */
4892 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
38807746
D		 4893
4894 /* allows RC6 residency counter to work */
4895 I915_WRITE(VLV_COUNTER_CONTROL,
4896 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
4897 VLV_MEDIA_RC6_COUNT_EN |
4898 VLV_RENDER_RC6_COUNT_EN));
4899
4900 /* For now we assume BIOS is allocating and populating the PCBR */
4901 pcbr = I915_READ(VLV_PCBR);
4902
38807746
D		 4903 /* 3: Enable RC6 */
4904 if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
4905 (pcbr >> VLV_PCBR_ADDR_SHIFT))
af5a75a3 4906 rc6_mode = GEN7_RC_CTL_TO_MODE;
38807746
D		 4907
4908 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
4909
2b6b3a09 4910 /* 4 Program defaults and thresholds for RPS*/
3cbdb48f 4911 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
2b6b3a09
D		 4912 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
4913 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
4914 I915_WRITE(GEN6_RP_UP_EI, 66000);
4915 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
4916
4917 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4918
4919 /* 5: Enable RPS */
4920 I915_WRITE(GEN6_RP_CONTROL,
4921 GEN6_RP_MEDIA_HW_NORMAL_MODE |
eb973a5e 4922 GEN6_RP_MEDIA_IS_GFX |
2b6b3a09
D		 4923 GEN6_RP_ENABLE |
4924 GEN6_RP_UP_BUSY_AVG |
4925 GEN6_RP_DOWN_IDLE_AVG);
4926
4927 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4928
8d40c3ae
VS		 4929 /* RPS code assumes GPLL is used */
4930 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
4931
c8e9627d 4932 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no");
2b6b3a09
D		 4933 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
4934
4935 dev_priv->rps.cur_freq = (val >> 8) & 0xff;
4936 DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
7c59a9c1 4937 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
2b6b3a09
D		 4938 dev_priv->rps.cur_freq);
4939
4940 DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
7c59a9c1 4941 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
2b6b3a09
D		 4942 dev_priv->rps.efficient_freq);
4943
4944 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
4945
59bad947 4946 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
38807746
D		 4947}
4948
0a073b84
JB		 4949static void valleyview_enable_rps(struct drm_device *dev)
4950{
4951 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 4952 struct intel_engine_cs *ring;
2a5913a8 4953 u32 gtfifodbg, val, rc6_mode = 0;
0a073b84
JB		 4954 int i;
4955
4956 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4957
ae48434c
ID		 4958 valleyview_check_pctx(dev_priv);
4959
0a073b84 4960 if ((gtfifodbg = I915_READ(GTFIFODBG))) {
f7d85c1e
JB		 4961 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
4962 gtfifodbg);
0a073b84
JB		 4963 I915_WRITE(GTFIFODBG, gtfifodbg);
4964 }
4965
c8d9a590 4966 /* If VLV, Forcewake all wells, else re-direct to regular path */
59bad947 4967 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
0a073b84 4968
160614a2
VS		 4969 /* Disable RC states. */
4970 I915_WRITE(GEN6_RC_CONTROL, 0);
4971
cad725fe 4972 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
0a073b84
JB		 4973 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
4974 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
4975 I915_WRITE(GEN6_RP_UP_EI, 66000);
4976 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
4977
4978 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4979
4980 I915_WRITE(GEN6_RP_CONTROL,
4981 GEN6_RP_MEDIA_TURBO |
4982 GEN6_RP_MEDIA_HW_NORMAL_MODE |
4983 GEN6_RP_MEDIA_IS_GFX |
4984 GEN6_RP_ENABLE |
4985 GEN6_RP_UP_BUSY_AVG |
4986 GEN6_RP_DOWN_IDLE_CONT);
4987
4988 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
4989 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
4990 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
4991
4992 for_each_ring(ring, dev_priv, i)
4993 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4994
2f0aa304 4995 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
0a073b84
JB		 4996
4997 /* allows RC6 residency counter to work */
49798eb2 4998 I915_WRITE(VLV_COUNTER_CONTROL,
31685c25
D		 4999 _MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN |
5000 VLV_RENDER_RC0_COUNT_EN |
49798eb2
JB		 5001 VLV_MEDIA_RC6_COUNT_EN |
5002 VLV_RENDER_RC6_COUNT_EN));
31685c25 5003
a2b23fe0 5004 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
6b88f295 5005 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
dc39fff7
BW		 5006
5007 intel_print_rc6_info(dev, rc6_mode);
5008
a2b23fe0 5009 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
0a073b84 5010
64936258 5011 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
0a073b84 5012
8d40c3ae
VS		 5013 /* RPS code assumes GPLL is used */
5014 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
5015
c8e9627d 5016 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no");
0a073b84
JB		 5017 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
5018
b39fb297 5019 dev_priv->rps.cur_freq = (val >> 8) & 0xff;
73008b98 5020 DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
7c59a9c1 5021 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
b39fb297 5022 dev_priv->rps.cur_freq);
0a073b84 5023
73008b98 5024 DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
7c59a9c1 5025 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
b39fb297 5026 dev_priv->rps.efficient_freq);
0a073b84 5027
b39fb297 5028 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
0a073b84 5029
59bad947 5030 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
0a073b84
JB		 5031}
5032
dde18883
ED		 5033static unsigned long intel_pxfreq(u32 vidfreq)
5034{
5035 unsigned long freq;
5036 int div = (vidfreq & 0x3f0000) >> 16;
5037 int post = (vidfreq & 0x3000) >> 12;
5038 int pre = (vidfreq & 0x7);
5039
5040 if (!pre)
5041 return 0;
5042
5043 freq = ((div * 133333) / ((1<<post) * pre));
5044
5045 return freq;
5046}
5047
eb48eb00
DV		 5048static const struct cparams {
5049 u16 i;
5050 u16 t;
5051 u16 m;
5052 u16 c;
5053} cparams[] = {
5054 { 1, 1333, 301, 28664 },
5055 { 1, 1066, 294, 24460 },
5056 { 1, 800, 294, 25192 },
5057 { 0, 1333, 276, 27605 },
5058 { 0, 1066, 276, 27605 },
5059 { 0, 800, 231, 23784 },
5060};
5061
f531dcb2 5062static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
eb48eb00
DV		 5063{
5064 u64 total_count, diff, ret;
5065 u32 count1, count2, count3, m = 0, c = 0;
5066 unsigned long now = jiffies_to_msecs(jiffies), diff1;
5067 int i;
5068
02d71956
DV		 5069 assert_spin_locked(&mchdev_lock);
5070
20e4d407 5071 diff1 = now - dev_priv->ips.last_time1;
eb48eb00
DV		 5072
5073 /* Prevent division-by-zero if we are asking too fast.
5074 * Also, we don't get interesting results if we are polling
5075 * faster than once in 10ms, so just return the saved value
5076 * in such cases.
5077 */
5078 if (diff1 <= 10)
20e4d407 5079 return dev_priv->ips.chipset_power;
eb48eb00
DV		 5080
5081 count1 = I915_READ(DMIEC);
5082 count2 = I915_READ(DDREC);
5083 count3 = I915_READ(CSIEC);
5084
5085 total_count = count1 + count2 + count3;
5086
5087 /* FIXME: handle per-counter overflow */
20e4d407
DV		 5088 if (total_count < dev_priv->ips.last_count1) {
5089 diff = ~0UL - dev_priv->ips.last_count1;
eb48eb00
DV		 5090 diff += total_count;
5091 } else {
20e4d407 5092 diff = total_count - dev_priv->ips.last_count1;
eb48eb00
DV		 5093 }
5094
5095 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
20e4d407
DV		 5096 if (cparams[i].i == dev_priv->ips.c_m &&
5097 cparams[i].t == dev_priv->ips.r_t) {
eb48eb00
DV		 5098 m = cparams[i].m;
5099 c = cparams[i].c;
5100 break;
5101 }
5102 }
5103
5104 diff = div_u64(diff, diff1);
5105 ret = ((m * diff) + c);
5106 ret = div_u64(ret, 10);
5107
20e4d407
DV		 5108 dev_priv->ips.last_count1 = total_count;
5109 dev_priv->ips.last_time1 = now;
eb48eb00 5110
20e4d407 5111 dev_priv->ips.chipset_power = ret;
eb48eb00
DV		 5112
5113 return ret;
5114}
5115
f531dcb2
CW		 5116unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
5117{
3d13ef2e 5118 struct drm_device *dev = dev_priv->dev;
f531dcb2
CW		 5119 unsigned long val;
5120
3d13ef2e 5121 if (INTEL_INFO(dev)->gen != 5)
f531dcb2
CW		 5122 return 0;
5123
5124 spin_lock_irq(&mchdev_lock);
5125
5126 val = __i915_chipset_val(dev_priv);
5127
5128 spin_unlock_irq(&mchdev_lock);
5129
5130 return val;
5131}
5132
eb48eb00
DV		 5133unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
5134{
5135 unsigned long m, x, b;
5136 u32 tsfs;
5137
5138 tsfs = I915_READ(TSFS);
5139
5140 m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
5141 x = I915_READ8(TR1);
5142
5143 b = tsfs & TSFS_INTR_MASK;
5144
5145 return ((m * x) / 127) - b;
5146}
5147
d972d6ee
MK		 5148static int _pxvid_to_vd(u8 pxvid)
5149{
5150 if (pxvid == 0)
5151 return 0;
5152
5153 if (pxvid >= 8 && pxvid < 31)
5154 pxvid = 31;
5155
5156 return (pxvid + 2) * 125;
5157}
5158
5159static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
eb48eb00 5160{
3d13ef2e 5161 struct drm_device *dev = dev_priv->dev;
d972d6ee
MK		 5162 const int vd = _pxvid_to_vd(pxvid);
5163 const int vm = vd - 1125;
5164
3d13ef2e 5165 if (INTEL_INFO(dev)->is_mobile)
d972d6ee
MK		 5166 return vm > 0 ? vm : 0;
5167
5168 return vd;
eb48eb00
DV		 5169}
5170
02d71956 5171static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
eb48eb00 5172{
5ed0bdf2 5173 u64 now, diff, diffms;
eb48eb00
DV		 5174 u32 count;
5175
02d71956 5176 assert_spin_locked(&mchdev_lock);
eb48eb00 5177
5ed0bdf2
TG		 5178 now = ktime_get_raw_ns();
5179 diffms = now - dev_priv->ips.last_time2;
5180 do_div(diffms, NSEC_PER_MSEC);
eb48eb00
DV		 5181
5182 /* Don't divide by 0 */
eb48eb00
DV		 5183 if (!diffms)
5184 return;
5185
5186 count = I915_READ(GFXEC);
5187
20e4d407
DV		 5188 if (count < dev_priv->ips.last_count2) {
5189 diff = ~0UL - dev_priv->ips.last_count2;
eb48eb00
DV		 5190 diff += count;
5191 } else {
20e4d407 5192 diff = count - dev_priv->ips.last_count2;
eb48eb00
DV		 5193 }
5194
20e4d407
DV		 5195 dev_priv->ips.last_count2 = count;
5196 dev_priv->ips.last_time2 = now;
eb48eb00
DV		 5197
5198 /* More magic constants... */
5199 diff = diff * 1181;
5200 diff = div_u64(diff, diffms * 10);
20e4d407 5201 dev_priv->ips.gfx_power = diff;
eb48eb00
DV		 5202}
5203
02d71956
DV		 5204void i915_update_gfx_val(struct drm_i915_private *dev_priv)
5205{
3d13ef2e
DL		 5206 struct drm_device *dev = dev_priv->dev;
5207
5208 if (INTEL_INFO(dev)->gen != 5)
02d71956
DV		 5209 return;
5210
9270388e 5211 spin_lock_irq(&mchdev_lock);
02d71956
DV		 5212
5213 __i915_update_gfx_val(dev_priv);
5214
9270388e 5215 spin_unlock_irq(&mchdev_lock);
02d71956
DV		 5216}
5217
f531dcb2 5218static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
eb48eb00
DV		 5219{
5220 unsigned long t, corr, state1, corr2, state2;
5221 u32 pxvid, ext_v;
5222
02d71956
DV		 5223 assert_spin_locked(&mchdev_lock);
5224
b39fb297 5225 pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4));
eb48eb00
DV		 5226 pxvid = (pxvid >> 24) & 0x7f;
5227 ext_v = pvid_to_extvid(dev_priv, pxvid);
5228
5229 state1 = ext_v;
5230
5231 t = i915_mch_val(dev_priv);
5232
5233 /* Revel in the empirically derived constants */
5234
5235 /* Correction factor in 1/100000 units */
5236 if (t > 80)
5237 corr = ((t * 2349) + 135940);
5238 else if (t >= 50)
5239 corr = ((t * 964) + 29317);
5240 else /* < 50 */
5241 corr = ((t * 301) + 1004);
5242
5243 corr = corr * ((150142 * state1) / 10000 - 78642);
5244 corr /= 100000;
20e4d407 5245 corr2 = (corr * dev_priv->ips.corr);
eb48eb00
DV		 5246
5247 state2 = (corr2 * state1) / 10000;
5248 state2 /= 100; /* convert to mW */
5249
02d71956 5250 __i915_update_gfx_val(dev_priv);
eb48eb00 5251
20e4d407 5252 return dev_priv->ips.gfx_power + state2;
eb48eb00
DV		 5253}
5254
f531dcb2
CW		 5255unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
5256{
3d13ef2e 5257 struct drm_device *dev = dev_priv->dev;
f531dcb2
CW		 5258 unsigned long val;
5259
3d13ef2e 5260 if (INTEL_INFO(dev)->gen != 5)
f531dcb2
CW		 5261 return 0;
5262
5263 spin_lock_irq(&mchdev_lock);
5264
5265 val = __i915_gfx_val(dev_priv);
5266
5267 spin_unlock_irq(&mchdev_lock);
5268
5269 return val;
5270}
5271
eb48eb00
DV		 5272/**
5273 * i915_read_mch_val - return value for IPS use
5274 *
5275 * Calculate and return a value for the IPS driver to use when deciding whether
5276 * we have thermal and power headroom to increase CPU or GPU power budget.
5277 */
5278unsigned long i915_read_mch_val(void)
5279{
5280 struct drm_i915_private *dev_priv;
5281 unsigned long chipset_val, graphics_val, ret = 0;
5282
9270388e 5283 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 5284 if (!i915_mch_dev)
5285 goto out_unlock;
5286 dev_priv = i915_mch_dev;
5287
f531dcb2
CW		 5288 chipset_val = __i915_chipset_val(dev_priv);
5289 graphics_val = __i915_gfx_val(dev_priv);
eb48eb00
DV		 5290
5291 ret = chipset_val + graphics_val;
5292
5293out_unlock:
9270388e 5294 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 5295
5296 return ret;
5297}
5298EXPORT_SYMBOL_GPL(i915_read_mch_val);
5299
5300/**
5301 * i915_gpu_raise - raise GPU frequency limit
5302 *
5303 * Raise the limit; IPS indicates we have thermal headroom.
5304 */
5305bool i915_gpu_raise(void)
5306{
5307 struct drm_i915_private *dev_priv;
5308 bool ret = true;
5309
9270388e 5310 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 5311 if (!i915_mch_dev) {
5312 ret = false;
5313 goto out_unlock;
5314 }
5315 dev_priv = i915_mch_dev;
5316
20e4d407
DV		 5317 if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
5318 dev_priv->ips.max_delay--;
eb48eb00
DV		 5319
5320out_unlock:
9270388e 5321 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 5322
5323 return ret;
5324}
5325EXPORT_SYMBOL_GPL(i915_gpu_raise);
5326
5327/**
5328 * i915_gpu_lower - lower GPU frequency limit
5329 *
5330 * IPS indicates we're close to a thermal limit, so throttle back the GPU
5331 * frequency maximum.
5332 */
5333bool i915_gpu_lower(void)
5334{
5335 struct drm_i915_private *dev_priv;
5336 bool ret = true;
5337
9270388e 5338 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 5339 if (!i915_mch_dev) {
5340 ret = false;
5341 goto out_unlock;
5342 }
5343 dev_priv = i915_mch_dev;
5344
20e4d407
DV		 5345 if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
5346 dev_priv->ips.max_delay++;
eb48eb00
DV		 5347
5348out_unlock:
9270388e 5349 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 5350
5351 return ret;
5352}
5353EXPORT_SYMBOL_GPL(i915_gpu_lower);
5354
5355/**
5356 * i915_gpu_busy - indicate GPU business to IPS
5357 *
5358 * Tell the IPS driver whether or not the GPU is busy.
5359 */
5360bool i915_gpu_busy(void)
5361{
5362 struct drm_i915_private *dev_priv;
a4872ba6 5363 struct intel_engine_cs *ring;
eb48eb00 5364 bool ret = false;
f047e395 5365 int i;
eb48eb00 5366
9270388e 5367 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 5368 if (!i915_mch_dev)
5369 goto out_unlock;
5370 dev_priv = i915_mch_dev;
5371
f047e395
CW		 5372 for_each_ring(ring, dev_priv, i)
5373 ret |= !list_empty(&ring->request_list);
eb48eb00
DV		 5374
5375out_unlock:
9270388e 5376 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 5377
5378 return ret;
5379}
5380EXPORT_SYMBOL_GPL(i915_gpu_busy);
5381
5382/**
5383 * i915_gpu_turbo_disable - disable graphics turbo
5384 *
5385 * Disable graphics turbo by resetting the max frequency and setting the
5386 * current frequency to the default.
5387 */
5388bool i915_gpu_turbo_disable(void)
5389{
5390 struct drm_i915_private *dev_priv;
5391 bool ret = true;
5392
9270388e 5393 spin_lock_irq(&mchdev_lock);
eb48eb00
DV		 5394 if (!i915_mch_dev) {
5395 ret = false;
5396 goto out_unlock;
5397 }
5398 dev_priv = i915_mch_dev;
5399
20e4d407 5400 dev_priv->ips.max_delay = dev_priv->ips.fstart;
eb48eb00 5401
20e4d407 5402 if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
eb48eb00
DV		 5403 ret = false;
5404
5405out_unlock:
9270388e 5406 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 5407
5408 return ret;
5409}
5410EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
5411
5412/**
5413 * Tells the intel_ips driver that the i915 driver is now loaded, if
5414 * IPS got loaded first.
5415 *
5416 * This awkward dance is so that neither module has to depend on the
5417 * other in order for IPS to do the appropriate communication of
5418 * GPU turbo limits to i915.
5419 */
5420static void
5421ips_ping_for_i915_load(void)
5422{
5423 void (*link)(void);
5424
5425 link = symbol_get(ips_link_to_i915_driver);
5426 if (link) {
5427 link();
5428 symbol_put(ips_link_to_i915_driver);
5429 }
5430}
5431
5432void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
5433{
02d71956
DV		 5434 /* We only register the i915 ips part with intel-ips once everything is
5435 * set up, to avoid intel-ips sneaking in and reading bogus values. */
9270388e 5436 spin_lock_irq(&mchdev_lock);
eb48eb00 5437 i915_mch_dev = dev_priv;
9270388e 5438 spin_unlock_irq(&mchdev_lock);
eb48eb00
DV		 5439
5440 ips_ping_for_i915_load();
5441}
5442
5443void intel_gpu_ips_teardown(void)
5444{
9270388e 5445 spin_lock_irq(&mchdev_lock);
eb48eb00 5446 i915_mch_dev = NULL;
9270388e 5447 spin_unlock_irq(&mchdev_lock);
eb48eb00 5448}
76c3552f 5449
8090c6b9 5450static void intel_init_emon(struct drm_device *dev)
dde18883
ED		 5451{
5452 struct drm_i915_private *dev_priv = dev->dev_private;
5453 u32 lcfuse;
5454 u8 pxw[16];
5455 int i;
5456
5457 /* Disable to program */
5458 I915_WRITE(ECR, 0);
5459 POSTING_READ(ECR);
5460
5461 /* Program energy weights for various events */
5462 I915_WRITE(SDEW, 0x15040d00);
5463 I915_WRITE(CSIEW0, 0x007f0000);
5464 I915_WRITE(CSIEW1, 0x1e220004);
5465 I915_WRITE(CSIEW2, 0x04000004);
5466
5467 for (i = 0; i < 5; i++)
5468 I915_WRITE(PEW + (i * 4), 0);
5469 for (i = 0; i < 3; i++)
5470 I915_WRITE(DEW + (i * 4), 0);
5471
5472 /* Program P-state weights to account for frequency power adjustment */
5473 for (i = 0; i < 16; i++) {
5474 u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
5475 unsigned long freq = intel_pxfreq(pxvidfreq);
5476 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
5477 PXVFREQ_PX_SHIFT;
5478 unsigned long val;
5479
5480 val = vid * vid;
5481 val *= (freq / 1000);
5482 val *= 255;
5483 val /= (127*127*900);
5484 if (val > 0xff)
5485 DRM_ERROR("bad pxval: %ld\n", val);
5486 pxw[i] = val;
5487 }
5488 /* Render standby states get 0 weight */
5489 pxw[14] = 0;
5490 pxw[15] = 0;
5491
5492 for (i = 0; i < 4; i++) {
5493 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
5494 (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
5495 I915_WRITE(PXW + (i * 4), val);
5496 }
5497
5498 /* Adjust magic regs to magic values (more experimental results) */
5499 I915_WRITE(OGW0, 0);
5500 I915_WRITE(OGW1, 0);
5501 I915_WRITE(EG0, 0x00007f00);
5502 I915_WRITE(EG1, 0x0000000e);
5503 I915_WRITE(EG2, 0x000e0000);
5504 I915_WRITE(EG3, 0x68000300);
5505 I915_WRITE(EG4, 0x42000000);
5506 I915_WRITE(EG5, 0x00140031);
5507 I915_WRITE(EG6, 0);
5508 I915_WRITE(EG7, 0);
5509
5510 for (i = 0; i < 8; i++)
5511 I915_WRITE(PXWL + (i * 4), 0);
5512
5513 /* Enable PMON + select events */
5514 I915_WRITE(ECR, 0x80000019);
5515
5516 lcfuse = I915_READ(LCFUSE02);
5517
20e4d407 5518 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
dde18883
ED		 5519}
5520
ae48434c
ID		 5521void intel_init_gt_powersave(struct drm_device *dev)
5522{
e6069ca8
ID		 5523 i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
5524
38807746
D		 5525 if (IS_CHERRYVIEW(dev))
5526 cherryview_init_gt_powersave(dev);
5527 else if (IS_VALLEYVIEW(dev))
4e80519e 5528 valleyview_init_gt_powersave(dev);
ae48434c
ID		 5529}
5530
5531void intel_cleanup_gt_powersave(struct drm_device *dev)
5532{
38807746
D		 5533 if (IS_CHERRYVIEW(dev))
5534 return;
5535 else if (IS_VALLEYVIEW(dev))
4e80519e 5536 valleyview_cleanup_gt_powersave(dev);
ae48434c
ID		 5537}
5538
dbea3cea
ID		 5539static void gen6_suspend_rps(struct drm_device *dev)
5540{
5541 struct drm_i915_private *dev_priv = dev->dev_private;
5542
5543 flush_delayed_work(&dev_priv->rps.delayed_resume_work);
5544
5545 /*
5546 * TODO: disable RPS interrupts on GEN9+ too once RPS support
5547 * is added for it.
5548 */
5549 if (INTEL_INFO(dev)->gen < 9)
5550 gen6_disable_rps_interrupts(dev);
5551}
5552
156c7ca0
JB		 5553/**
5554 * intel_suspend_gt_powersave - suspend PM work and helper threads
5555 * @dev: drm device
5556 *
5557 * We don't want to disable RC6 or other features here, we just want
5558 * to make sure any work we've queued has finished and won't bother
5559 * us while we're suspended.
5560 */
5561void intel_suspend_gt_powersave(struct drm_device *dev)
5562{
5563 struct drm_i915_private *dev_priv = dev->dev_private;
5564
d4d70aa5
ID		 5565 if (INTEL_INFO(dev)->gen < 6)
5566 return;
5567
dbea3cea 5568 gen6_suspend_rps(dev);
b47adc17
D		 5569
5570 /* Force GPU to min freq during suspend */
5571 gen6_rps_idle(dev_priv);
156c7ca0
JB		 5572}
5573
8090c6b9
DV		 5574void intel_disable_gt_powersave(struct drm_device *dev)
5575{
1a01ab3b
JB		 5576 struct drm_i915_private *dev_priv = dev->dev_private;
5577
930ebb46 5578 if (IS_IRONLAKE_M(dev)) {
8090c6b9 5579 ironlake_disable_drps(dev);
38807746 5580 } else if (INTEL_INFO(dev)->gen >= 6) {
10d8d366 5581 intel_suspend_gt_powersave(dev);
e494837a 5582
4fc688ce 5583 mutex_lock(&dev_priv->rps.hw_lock);
20e49366
ZW		 5584 if (INTEL_INFO(dev)->gen >= 9)
5585 gen9_disable_rps(dev);
5586 else if (IS_CHERRYVIEW(dev))
38807746
D		 5587 cherryview_disable_rps(dev);
5588 else if (IS_VALLEYVIEW(dev))
d20d4f0c
JB		 5589 valleyview_disable_rps(dev);
5590 else
5591 gen6_disable_rps(dev);
e534770a 5592
c0951f0c 5593 dev_priv->rps.enabled = false;
4fc688ce 5594 mutex_unlock(&dev_priv->rps.hw_lock);
930ebb46 5595 }
8090c6b9
DV		 5596}
5597
1a01ab3b
JB		 5598static void intel_gen6_powersave_work(struct work_struct *work)
5599{
5600 struct drm_i915_private *dev_priv =
5601 container_of(work, struct drm_i915_private,
5602 rps.delayed_resume_work.work);
5603 struct drm_device *dev = dev_priv->dev;
5604
4fc688ce 5605 mutex_lock(&dev_priv->rps.hw_lock);
0a073b84 5606
3cc134e3
ID		 5607 /*
5608 * TODO: reset/enable RPS interrupts on GEN9+ too, once RPS support is
5609 * added for it.
5610 */
5611 if (INTEL_INFO(dev)->gen < 9)
5612 gen6_reset_rps_interrupts(dev);
5613
38807746
D		 5614 if (IS_CHERRYVIEW(dev)) {
5615 cherryview_enable_rps(dev);
5616 } else if (IS_VALLEYVIEW(dev)) {
0a073b84 5617 valleyview_enable_rps(dev);
20e49366 5618 } else if (INTEL_INFO(dev)->gen >= 9) {
b6fef0ef 5619 gen9_enable_rc6(dev);
20e49366 5620 gen9_enable_rps(dev);
b6fef0ef 5621 __gen6_update_ring_freq(dev);
6edee7f3
BW		 5622 } else if (IS_BROADWELL(dev)) {
5623 gen8_enable_rps(dev);
c2bc2fc5 5624 __gen6_update_ring_freq(dev);
0a073b84
JB		 5625 } else {
5626 gen6_enable_rps(dev);
c2bc2fc5 5627 __gen6_update_ring_freq(dev);
0a073b84 5628 }
c0951f0c 5629 dev_priv->rps.enabled = true;
3cc134e3
ID		 5630
5631 if (INTEL_INFO(dev)->gen < 9)
5632 gen6_enable_rps_interrupts(dev);
5633
4fc688ce 5634 mutex_unlock(&dev_priv->rps.hw_lock);
c6df39b5
ID		 5635
5636 intel_runtime_pm_put(dev_priv);
1a01ab3b
JB		 5637}
5638
8090c6b9
DV		 5639void intel_enable_gt_powersave(struct drm_device *dev)
5640{
1a01ab3b
JB		 5641 struct drm_i915_private *dev_priv = dev->dev_private;
5642
f61018b1
YZ		 5643 /* Powersaving is controlled by the host when inside a VM */
5644 if (intel_vgpu_active(dev))
5645 return;
5646
8090c6b9 5647 if (IS_IRONLAKE_M(dev)) {
dc1d0136 5648 mutex_lock(&dev->struct_mutex);
8090c6b9 5649 ironlake_enable_drps(dev);
8090c6b9 5650 intel_init_emon(dev);
dc1d0136 5651 mutex_unlock(&dev->struct_mutex);
38807746 5652 } else if (INTEL_INFO(dev)->gen >= 6) {
1a01ab3b
JB		 5653 /*
5654 * PCU communication is slow and this doesn't need to be
5655 * done at any specific time, so do this out of our fast path
5656 * to make resume and init faster.
c6df39b5
ID		 5657 *
5658 * We depend on the HW RC6 power context save/restore
5659 * mechanism when entering D3 through runtime PM suspend. So
5660 * disable RPM until RPS/RC6 is properly setup. We can only
5661 * get here via the driver load/system resume/runtime resume
5662 * paths, so the _noresume version is enough (and in case of
5663 * runtime resume it's necessary).
1a01ab3b 5664 */
c6df39b5
ID		 5665 if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
5666 round_jiffies_up_relative(HZ)))
5667 intel_runtime_pm_get_noresume(dev_priv);
8090c6b9
DV		 5668 }
5669}
5670
c6df39b5
ID		 5671void intel_reset_gt_powersave(struct drm_device *dev)
5672{
5673 struct drm_i915_private *dev_priv = dev->dev_private;
5674
dbea3cea
ID		 5675 if (INTEL_INFO(dev)->gen < 6)
5676 return;
5677
5678 gen6_suspend_rps(dev);
c6df39b5 5679 dev_priv->rps.enabled = false;
c6df39b5
ID		 5680}
5681
3107bd48
DV		 5682static void ibx_init_clock_gating(struct drm_device *dev)
5683{
5684 struct drm_i915_private *dev_priv = dev->dev_private;
5685
5686 /*
5687 * On Ibex Peak and Cougar Point, we need to disable clock
5688 * gating for the panel power sequencer or it will fail to
5689 * start up when no ports are active.
5690 */
5691 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
5692}
5693
0e088b8f
VS		 5694static void g4x_disable_trickle_feed(struct drm_device *dev)
5695{
5696 struct drm_i915_private *dev_priv = dev->dev_private;
5697 int pipe;
5698
055e393f 5699 for_each_pipe(dev_priv, pipe) {
0e088b8f
VS		 5700 I915_WRITE(DSPCNTR(pipe),
5701 I915_READ(DSPCNTR(pipe)) |
5702 DISPPLANE_TRICKLE_FEED_DISABLE);
1dba99f4 5703 intel_flush_primary_plane(dev_priv, pipe);
0e088b8f
VS		 5704 }
5705}
5706
017636cc
VS		 5707static void ilk_init_lp_watermarks(struct drm_device *dev)
5708{
5709 struct drm_i915_private *dev_priv = dev->dev_private;
5710
5711 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
5712 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
5713 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
5714
5715 /*
5716 * Don't touch WM1S_LP_EN here.
5717 * Doing so could cause underruns.
5718 */
5719}
5720
1fa61106 5721static void ironlake_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5722{
5723 struct drm_i915_private *dev_priv = dev->dev_private;
231e54f6 5724 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6f1d69b0 5725
f1e8fa56
DL		 5726 /*
5727 * Required for FBC
5728 * WaFbcDisableDpfcClockGating:ilk
5729 */
4d47e4f5
DL		 5730 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
5731 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
5732 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6f1d69b0
ED		 5733
5734 I915_WRITE(PCH_3DCGDIS0,
5735 MARIUNIT_CLOCK_GATE_DISABLE |
5736 SVSMUNIT_CLOCK_GATE_DISABLE);
5737 I915_WRITE(PCH_3DCGDIS1,
5738 VFMUNIT_CLOCK_GATE_DISABLE);
5739
6f1d69b0
ED		 5740 /*
5741 * According to the spec the following bits should be set in
5742 * order to enable memory self-refresh
5743 * The bit 22/21 of 0x42004
5744 * The bit 5 of 0x42020
5745 * The bit 15 of 0x45000
5746 */
5747 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5748 (I915_READ(ILK_DISPLAY_CHICKEN2) |
5749 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4d47e4f5 5750 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6f1d69b0
ED		 5751 I915_WRITE(DISP_ARB_CTL,
5752 (I915_READ(DISP_ARB_CTL) |
5753 DISP_FBC_WM_DIS));
017636cc
VS		 5754
5755 ilk_init_lp_watermarks(dev);
6f1d69b0
ED		 5756
5757 /*
5758 * Based on the document from hardware guys the following bits
5759 * should be set unconditionally in order to enable FBC.
5760 * The bit 22 of 0x42000
5761 * The bit 22 of 0x42004
5762 * The bit 7,8,9 of 0x42020.
5763 */
5764 if (IS_IRONLAKE_M(dev)) {
4bb35334 5765 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6f1d69b0
ED		 5766 I915_WRITE(ILK_DISPLAY_CHICKEN1,
5767 I915_READ(ILK_DISPLAY_CHICKEN1) |
5768 ILK_FBCQ_DIS);
5769 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5770 I915_READ(ILK_DISPLAY_CHICKEN2) |
5771 ILK_DPARB_GATE);
6f1d69b0
ED		 5772 }
5773
4d47e4f5
DL		 5774 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
5775
6f1d69b0
ED		 5776 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5777 I915_READ(ILK_DISPLAY_CHICKEN2) |
5778 ILK_ELPIN_409_SELECT);
5779 I915_WRITE(_3D_CHICKEN2,
5780 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
5781 _3D_CHICKEN2_WM_READ_PIPELINED);
4358a374 5782
ecdb4eb7 5783 /* WaDisableRenderCachePipelinedFlush:ilk */
4358a374
DV		 5784 I915_WRITE(CACHE_MODE_0,
5785 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
3107bd48 5786
4e04632e
AG		 5787 /* WaDisable_RenderCache_OperationalFlush:ilk */
5788 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5789
0e088b8f 5790 g4x_disable_trickle_feed(dev);
bdad2b2f 5791
3107bd48
DV		 5792 ibx_init_clock_gating(dev);
5793}
5794
5795static void cpt_init_clock_gating(struct drm_device *dev)
5796{
5797 struct drm_i915_private *dev_priv = dev->dev_private;
5798 int pipe;
3f704fa2 5799 uint32_t val;
3107bd48
DV		 5800
5801 /*
5802 * On Ibex Peak and Cougar Point, we need to disable clock
5803 * gating for the panel power sequencer or it will fail to
5804 * start up when no ports are active.
5805 */
cd664078
JB		 5806 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
5807 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
5808 PCH_CPUNIT_CLOCK_GATE_DISABLE);
3107bd48
DV		 5809 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
5810 DPLS_EDP_PPS_FIX_DIS);
335c07b7
TI		 5811 /* The below fixes the weird display corruption, a few pixels shifted
5812 * downward, on (only) LVDS of some HP laptops with IVY.
5813 */
055e393f 5814 for_each_pipe(dev_priv, pipe) {
dc4bd2d1
PZ		 5815 val = I915_READ(TRANS_CHICKEN2(pipe));
5816 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
5817 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
41aa3448 5818 if (dev_priv->vbt.fdi_rx_polarity_inverted)
3f704fa2 5819 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
dc4bd2d1
PZ		 5820 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
5821 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
5822 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
3f704fa2
PZ		 5823 I915_WRITE(TRANS_CHICKEN2(pipe), val);
5824 }
3107bd48 5825 /* WADP0ClockGatingDisable */
055e393f 5826 for_each_pipe(dev_priv, pipe) {
3107bd48
DV		 5827 I915_WRITE(TRANS_CHICKEN1(pipe),
5828 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
5829 }
6f1d69b0
ED		 5830}
5831
1d7aaa0c
DV		 5832static void gen6_check_mch_setup(struct drm_device *dev)
5833{
5834 struct drm_i915_private *dev_priv = dev->dev_private;
5835 uint32_t tmp;
5836
5837 tmp = I915_READ(MCH_SSKPD);
df662a28
DV		 5838 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
5839 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
5840 tmp);
1d7aaa0c
DV		 5841}
5842
1fa61106 5843static void gen6_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 5844{
5845 struct drm_i915_private *dev_priv = dev->dev_private;
231e54f6 5846 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6f1d69b0 5847
231e54f6 5848 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6f1d69b0
ED		 5849
5850 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5851 I915_READ(ILK_DISPLAY_CHICKEN2) |
5852 ILK_ELPIN_409_SELECT);
5853
ecdb4eb7 5854 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
4283908e
DV		 5855 I915_WRITE(_3D_CHICKEN,
5856 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
5857
4e04632e
AG		 5858 /* WaDisable_RenderCache_OperationalFlush:snb */
5859 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5860
8d85d272
VS		 5861 /*
5862 * BSpec recoomends 8x4 when MSAA is used,
5863 * however in practice 16x4 seems fastest.
c5c98a58
VS		 5864 *
5865 * Note that PS/WM thread counts depend on the WIZ hashing
5866 * disable bit, which we don't touch here, but it's good
5867 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
8d85d272
VS		 5868 */
5869 I915_WRITE(GEN6_GT_MODE,
98533251 5870 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8d85d272 5871
017636cc 5872 ilk_init_lp_watermarks(dev);
6f1d69b0 5873
6f1d69b0 5874 I915_WRITE(CACHE_MODE_0,
50743298 5875 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
6f1d69b0
ED		 5876
5877 I915_WRITE(GEN6_UCGCTL1,
5878 I915_READ(GEN6_UCGCTL1) |
5879 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
5880 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
5881
5882 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
5883 * gating disable must be set. Failure to set it results in
5884 * flickering pixels due to Z write ordering failures after
5885 * some amount of runtime in the Mesa "fire" demo, and Unigine
5886 * Sanctuary and Tropics, and apparently anything else with
5887 * alpha test or pixel discard.
5888 *
5889 * According to the spec, bit 11 (RCCUNIT) must also be set,
5890 * but we didn't debug actual testcases to find it out.
0f846f81 5891 *
ef59318c
VS		 5892 * WaDisableRCCUnitClockGating:snb
5893 * WaDisableRCPBUnitClockGating:snb
6f1d69b0
ED		 5894 */
5895 I915_WRITE(GEN6_UCGCTL2,
5896 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
5897 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
5898
5eb146dd 5899 /* WaStripsFansDisableFastClipPerformanceFix:snb */
743b57d8
VS		 5900 I915_WRITE(_3D_CHICKEN3,
5901 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
6f1d69b0 5902
e927ecde
VS		 5903 /*
5904 * Bspec says:
5905 * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
5906 * 3DSTATE_SF number of SF output attributes is more than 16."
5907 */
5908 I915_WRITE(_3D_CHICKEN3,
5909 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
5910
6f1d69b0
ED		 5911 /*
5912 * According to the spec the following bits should be
5913 * set in order to enable memory self-refresh and fbc:
5914 * The bit21 and bit22 of 0x42000
5915 * The bit21 and bit22 of 0x42004
5916 * The bit5 and bit7 of 0x42020
5917 * The bit14 of 0x70180
5918 * The bit14 of 0x71180
4bb35334
DL		 5919 *
5920 * WaFbcAsynchFlipDisableFbcQueue:snb
6f1d69b0
ED		 5921 */
5922 I915_WRITE(ILK_DISPLAY_CHICKEN1,
5923 I915_READ(ILK_DISPLAY_CHICKEN1) |
5924 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
5925 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5926 I915_READ(ILK_DISPLAY_CHICKEN2) |
5927 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
231e54f6
DL		 5928 I915_WRITE(ILK_DSPCLK_GATE_D,
5929 I915_READ(ILK_DSPCLK_GATE_D) |
5930 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
5931 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
6f1d69b0 5932
0e088b8f 5933 g4x_disable_trickle_feed(dev);
f8f2ac9a 5934
3107bd48 5935 cpt_init_clock_gating(dev);
1d7aaa0c
DV		 5936
5937 gen6_check_mch_setup(dev);
6f1d69b0
ED		 5938}
5939
5940static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
5941{
5942 uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
5943
3aad9059 5944 /*
46680e0a 5945 * WaVSThreadDispatchOverride:ivb,vlv
3aad9059
VS		 5946 *
5947 * This actually overrides the dispatch
5948 * mode for all thread types.
5949 */
6f1d69b0
ED		 5950 reg &= ~GEN7_FF_SCHED_MASK;
5951 reg |= GEN7_FF_TS_SCHED_HW;
5952 reg |= GEN7_FF_VS_SCHED_HW;
5953 reg |= GEN7_FF_DS_SCHED_HW;
5954
5955 I915_WRITE(GEN7_FF_THREAD_MODE, reg);
5956}
5957
17a303ec
PZ		 5958static void lpt_init_clock_gating(struct drm_device *dev)
5959{
5960 struct drm_i915_private *dev_priv = dev->dev_private;
5961
5962 /*
5963 * TODO: this bit should only be enabled when really needed, then
5964 * disabled when not needed anymore in order to save power.
5965 */
5966 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
5967 I915_WRITE(SOUTH_DSPCLK_GATE_D,
5968 I915_READ(SOUTH_DSPCLK_GATE_D) |
5969 PCH_LP_PARTITION_LEVEL_DISABLE);
0a790cdb
PZ		 5970
5971 /* WADPOClockGatingDisable:hsw */
5972 I915_WRITE(_TRANSA_CHICKEN1,
5973 I915_READ(_TRANSA_CHICKEN1) |
5974 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
17a303ec
PZ		 5975}
5976
7d708ee4
ID		 5977static void lpt_suspend_hw(struct drm_device *dev)
5978{
5979 struct drm_i915_private *dev_priv = dev->dev_private;
5980
5981 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
5982 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
5983
5984 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
5985 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
5986 }
5987}
5988
47c2bd97 5989static void broadwell_init_clock_gating(struct drm_device *dev)
1020a5c2
BW		 5990{
5991 struct drm_i915_private *dev_priv = dev->dev_private;
07d27e20 5992 enum pipe pipe;
1020a5c2
BW		 5993
5994 I915_WRITE(WM3_LP_ILK, 0);
5995 I915_WRITE(WM2_LP_ILK, 0);
5996 I915_WRITE(WM1_LP_ILK, 0);
50ed5fbd 5997
ab57fff1 5998 /* WaSwitchSolVfFArbitrationPriority:bdw */
50ed5fbd 5999 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
fe4ab3ce 6000
ab57fff1 6001 /* WaPsrDPAMaskVBlankInSRD:bdw */
fe4ab3ce
BW		 6002 I915_WRITE(CHICKEN_PAR1_1,
6003 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
6004
ab57fff1 6005 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
055e393f 6006 for_each_pipe(dev_priv, pipe) {
07d27e20 6007 I915_WRITE(CHICKEN_PIPESL_1(pipe),
c7c65622 6008 I915_READ(CHICKEN_PIPESL_1(pipe)) |
8f670bb1 6009 BDW_DPRS_MASK_VBLANK_SRD);
fe4ab3ce 6010 }
63801f21 6011
ab57fff1
BW		 6012 /* WaVSRefCountFullforceMissDisable:bdw */
6013 /* WaDSRefCountFullforceMissDisable:bdw */
6014 I915_WRITE(GEN7_FF_THREAD_MODE,
6015 I915_READ(GEN7_FF_THREAD_MODE) &
6016 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
36075a4c 6017
295e8bb7
VS		 6018 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6019 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4f1ca9e9
VS		 6020
6021 /* WaDisableSDEUnitClockGating:bdw */
6022 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6023 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
5d708680 6024
89d6b2b8 6025 lpt_init_clock_gating(dev);
1020a5c2
BW		 6026}
6027
cad2a2d7
ED		 6028static void haswell_init_clock_gating(struct drm_device *dev)
6029{
6030 struct drm_i915_private *dev_priv = dev->dev_private;
cad2a2d7 6031
017636cc 6032 ilk_init_lp_watermarks(dev);
cad2a2d7 6033
f3fc4884
FJ		 6034 /* L3 caching of data atomics doesn't work -- disable it. */
6035 I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
6036 I915_WRITE(HSW_ROW_CHICKEN3,
6037 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
6038
ecdb4eb7 6039 /* This is required by WaCatErrorRejectionIssue:hsw */
cad2a2d7
ED		 6040 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6041 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6042 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6043
e36ea7ff
VS		 6044 /* WaVSRefCountFullforceMissDisable:hsw */
6045 I915_WRITE(GEN7_FF_THREAD_MODE,
6046 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
cad2a2d7 6047
4e04632e
AG		 6048 /* WaDisable_RenderCache_OperationalFlush:hsw */
6049 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6050
fe27c606
CW		 6051 /* enable HiZ Raw Stall Optimization */
6052 I915_WRITE(CACHE_MODE_0_GEN7,
6053 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6054
ecdb4eb7 6055 /* WaDisable4x2SubspanOptimization:hsw */
cad2a2d7
ED		 6056 I915_WRITE(CACHE_MODE_1,
6057 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
1544d9d5 6058
a12c4967
VS		 6059 /*
6060 * BSpec recommends 8x4 when MSAA is used,
6061 * however in practice 16x4 seems fastest.
c5c98a58
VS		 6062 *
6063 * Note that PS/WM thread counts depend on the WIZ hashing
6064 * disable bit, which we don't touch here, but it's good
6065 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
a12c4967
VS		 6066 */
6067 I915_WRITE(GEN7_GT_MODE,
98533251 6068 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
a12c4967 6069
94411593
KG		 6070 /* WaSampleCChickenBitEnable:hsw */
6071 I915_WRITE(HALF_SLICE_CHICKEN3,
6072 _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
6073
ecdb4eb7 6074 /* WaSwitchSolVfFArbitrationPriority:hsw */
e3dff585
BW		 6075 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
6076
90a88643
PZ		 6077 /* WaRsPkgCStateDisplayPMReq:hsw */
6078 I915_WRITE(CHICKEN_PAR1_1,
6079 I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
1544d9d5 6080
17a303ec 6081 lpt_init_clock_gating(dev);
cad2a2d7
ED		 6082}
6083
1fa61106 6084static void ivybridge_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 6085{
6086 struct drm_i915_private *dev_priv = dev->dev_private;
20848223 6087 uint32_t snpcr;
6f1d69b0 6088
017636cc 6089 ilk_init_lp_watermarks(dev);
6f1d69b0 6090
231e54f6 6091 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
6f1d69b0 6092
ecdb4eb7 6093 /* WaDisableEarlyCull:ivb */
87f8020e
JB		 6094 I915_WRITE(_3D_CHICKEN3,
6095 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6096
ecdb4eb7 6097 /* WaDisableBackToBackFlipFix:ivb */
6f1d69b0
ED		 6098 I915_WRITE(IVB_CHICKEN3,
6099 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6100 CHICKEN3_DGMG_DONE_FIX_DISABLE);
6101
ecdb4eb7 6102 /* WaDisablePSDDualDispatchEnable:ivb */
12f3382b
JB		 6103 if (IS_IVB_GT1(dev))
6104 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
6105 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
12f3382b 6106
4e04632e
AG		 6107 /* WaDisable_RenderCache_OperationalFlush:ivb */
6108 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6109
ecdb4eb7 6110 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
6f1d69b0
ED		 6111 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
6112 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
6113
ecdb4eb7 6114 /* WaApplyL3ControlAndL3ChickenMode:ivb */
6f1d69b0
ED		 6115 I915_WRITE(GEN7_L3CNTLREG1,
6116 GEN7_WA_FOR_GEN7_L3_CONTROL);
6117 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
8ab43976
JB		 6118 GEN7_WA_L3_CHICKEN_MODE);
6119 if (IS_IVB_GT1(dev))
6120 I915_WRITE(GEN7_ROW_CHICKEN2,
6121 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
412236c2
VS		 6122 else {
6123 /* must write both registers */
6124 I915_WRITE(GEN7_ROW_CHICKEN2,
6125 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8ab43976
JB		 6126 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
6127 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
412236c2 6128 }
6f1d69b0 6129
ecdb4eb7 6130 /* WaForceL3Serialization:ivb */
61939d97
JB		 6131 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6132 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6133
1b80a19a 6134 /*
0f846f81 6135 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
ecdb4eb7 6136 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
0f846f81
JB		 6137 */
6138 I915_WRITE(GEN6_UCGCTL2,
28acf3b2 6139 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
0f846f81 6140
ecdb4eb7 6141 /* This is required by WaCatErrorRejectionIssue:ivb */
6f1d69b0
ED		 6142 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6143 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6144 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6145
0e088b8f 6146 g4x_disable_trickle_feed(dev);
6f1d69b0
ED		 6147
6148 gen7_setup_fixed_func_scheduler(dev_priv);
97e1930f 6149
22721343
CW		 6150 if (0) { /* causes HiZ corruption on ivb:gt1 */
6151 /* enable HiZ Raw Stall Optimization */
6152 I915_WRITE(CACHE_MODE_0_GEN7,
6153 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6154 }
116f2b6d 6155
ecdb4eb7 6156 /* WaDisable4x2SubspanOptimization:ivb */
97e1930f
DV		 6157 I915_WRITE(CACHE_MODE_1,
6158 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
20848223 6159
a607c1a4
VS		 6160 /*
6161 * BSpec recommends 8x4 when MSAA is used,
6162 * however in practice 16x4 seems fastest.
c5c98a58
VS		 6163 *
6164 * Note that PS/WM thread counts depend on the WIZ hashing
6165 * disable bit, which we don't touch here, but it's good
6166 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
a607c1a4
VS		 6167 */
6168 I915_WRITE(GEN7_GT_MODE,
98533251 6169 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
a607c1a4 6170
20848223
BW		 6171 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
6172 snpcr &= ~GEN6_MBC_SNPCR_MASK;
6173 snpcr |= GEN6_MBC_SNPCR_MED;
6174 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3107bd48 6175
ab5c608b
BW		 6176 if (!HAS_PCH_NOP(dev))
6177 cpt_init_clock_gating(dev);
1d7aaa0c
DV		 6178
6179 gen6_check_mch_setup(dev);
6f1d69b0
ED		 6180}
6181
c6beb13e
VS		 6182static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
6183{
6184 I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
6185
6186 /*
6187 * Disable trickle feed and enable pnd deadline calculation
6188 */
6189 I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
6190 I915_WRITE(CBR1_VLV, 0);
6191}
6192
1fa61106 6193static void valleyview_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 6194{
6195 struct drm_i915_private *dev_priv = dev->dev_private;
6f1d69b0 6196
c6beb13e 6197 vlv_init_display_clock_gating(dev_priv);
6f1d69b0 6198
ecdb4eb7 6199 /* WaDisableEarlyCull:vlv */
87f8020e
JB		 6200 I915_WRITE(_3D_CHICKEN3,
6201 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6202
ecdb4eb7 6203 /* WaDisableBackToBackFlipFix:vlv */
6f1d69b0
ED		 6204 I915_WRITE(IVB_CHICKEN3,
6205 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6206 CHICKEN3_DGMG_DONE_FIX_DISABLE);
6207
fad7d36e 6208 /* WaPsdDispatchEnable:vlv */
ecdb4eb7 6209 /* WaDisablePSDDualDispatchEnable:vlv */
12f3382b 6210 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
d3bc0303
JB		 6211 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
6212 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
12f3382b 6213
4e04632e
AG		 6214 /* WaDisable_RenderCache_OperationalFlush:vlv */
6215 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6216
ecdb4eb7 6217 /* WaForceL3Serialization:vlv */
61939d97
JB		 6218 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6219 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6220
ecdb4eb7 6221 /* WaDisableDopClockGating:vlv */
8ab43976
JB		 6222 I915_WRITE(GEN7_ROW_CHICKEN2,
6223 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6224
ecdb4eb7 6225 /* This is required by WaCatErrorRejectionIssue:vlv */
6f1d69b0
ED		 6226 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6227 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6228 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6229
46680e0a
VS		 6230 gen7_setup_fixed_func_scheduler(dev_priv);
6231
3c0edaeb 6232 /*
0f846f81 6233 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
ecdb4eb7 6234 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
0f846f81
JB		 6235 */
6236 I915_WRITE(GEN6_UCGCTL2,
3c0edaeb 6237 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
0f846f81 6238
c98f5062
AG		 6239 /* WaDisableL3Bank2xClockGate:vlv
6240 * Disabling L3 clock gating- MMIO 940c[25] = 1
6241 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
6242 I915_WRITE(GEN7_UCGCTL4,
6243 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
e3f33d46 6244
afd58e79
VS		 6245 /*
6246 * BSpec says this must be set, even though
6247 * WaDisable4x2SubspanOptimization isn't listed for VLV.
6248 */
6b26c86d
DV		 6249 I915_WRITE(CACHE_MODE_1,
6250 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
7983117f 6251
da2518f9
VS		 6252 /*
6253 * BSpec recommends 8x4 when MSAA is used,
6254 * however in practice 16x4 seems fastest.
6255 *
6256 * Note that PS/WM thread counts depend on the WIZ hashing
6257 * disable bit, which we don't touch here, but it's good
6258 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6259 */
6260 I915_WRITE(GEN7_GT_MODE,
6261 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6262
031994ee
VS		 6263 /*
6264 * WaIncreaseL3CreditsForVLVB0:vlv
6265 * This is the hardware default actually.
6266 */
6267 I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
6268
2d809570 6269 /*
ecdb4eb7 6270 * WaDisableVLVClockGating_VBIIssue:vlv
2d809570
JB		 6271 * Disable clock gating on th GCFG unit to prevent a delay
6272 * in the reporting of vblank events.
6273 */
7a0d1eed 6274 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
6f1d69b0
ED		 6275}
6276
a4565da8
VS		 6277static void cherryview_init_clock_gating(struct drm_device *dev)
6278{
6279 struct drm_i915_private *dev_priv = dev->dev_private;
6280
c6beb13e 6281 vlv_init_display_clock_gating(dev_priv);
dd811e70 6282
232ce337
VS		 6283 /* WaVSRefCountFullforceMissDisable:chv */
6284 /* WaDSRefCountFullforceMissDisable:chv */
6285 I915_WRITE(GEN7_FF_THREAD_MODE,
6286 I915_READ(GEN7_FF_THREAD_MODE) &
6287 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
acea6f95
VS		 6288
6289 /* WaDisableSemaphoreAndSyncFlipWait:chv */
6290 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6291 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
0846697c
VS		 6292
6293 /* WaDisableCSUnitClockGating:chv */
6294 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
6295 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
c631780f
VS		 6296
6297 /* WaDisableSDEUnitClockGating:chv */
6298 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6299 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
a4565da8
VS		 6300}
6301
1fa61106 6302static void g4x_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 6303{
6304 struct drm_i915_private *dev_priv = dev->dev_private;
6305 uint32_t dspclk_gate;
6306
6307 I915_WRITE(RENCLK_GATE_D1, 0);
6308 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
6309 GS_UNIT_CLOCK_GATE_DISABLE |
6310 CL_UNIT_CLOCK_GATE_DISABLE);
6311 I915_WRITE(RAMCLK_GATE_D, 0);
6312 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
6313 OVRUNIT_CLOCK_GATE_DISABLE |
6314 OVCUNIT_CLOCK_GATE_DISABLE;
6315 if (IS_GM45(dev))
6316 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
6317 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
4358a374
DV		 6318
6319 /* WaDisableRenderCachePipelinedFlush */
6320 I915_WRITE(CACHE_MODE_0,
6321 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
de1aa629 6322
4e04632e
AG		 6323 /* WaDisable_RenderCache_OperationalFlush:g4x */
6324 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6325
0e088b8f 6326 g4x_disable_trickle_feed(dev);
6f1d69b0
ED		 6327}
6328
1fa61106 6329static void crestline_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 6330{
6331 struct drm_i915_private *dev_priv = dev->dev_private;
6332
6333 I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
6334 I915_WRITE(RENCLK_GATE_D2, 0);
6335 I915_WRITE(DSPCLK_GATE_D, 0);
6336 I915_WRITE(RAMCLK_GATE_D, 0);
6337 I915_WRITE16(DEUC, 0);
20f94967
VS		 6338 I915_WRITE(MI_ARB_STATE,
6339 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4e04632e
AG		 6340
6341 /* WaDisable_RenderCache_OperationalFlush:gen4 */
6342 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6f1d69b0
ED		 6343}
6344
1fa61106 6345static void broadwater_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 6346{
6347 struct drm_i915_private *dev_priv = dev->dev_private;
6348
6349 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
6350 I965_RCC_CLOCK_GATE_DISABLE |
6351 I965_RCPB_CLOCK_GATE_DISABLE |
6352 I965_ISC_CLOCK_GATE_DISABLE |
6353 I965_FBC_CLOCK_GATE_DISABLE);
6354 I915_WRITE(RENCLK_GATE_D2, 0);
20f94967
VS		 6355 I915_WRITE(MI_ARB_STATE,
6356 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4e04632e
AG		 6357
6358 /* WaDisable_RenderCache_OperationalFlush:gen4 */
6359 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6f1d69b0
ED		 6360}
6361
1fa61106 6362static void gen3_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 6363{
6364 struct drm_i915_private *dev_priv = dev->dev_private;
6365 u32 dstate = I915_READ(D_STATE);
6366
6367 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
6368 DSTATE_DOT_CLOCK_GATING;
6369 I915_WRITE(D_STATE, dstate);
13a86b85
CW		 6370
6371 if (IS_PINEVIEW(dev))
6372 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
974a3b0f
DV		 6373
6374 /* IIR "flip pending" means done if this bit is set */
6375 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
12fabbcb
VS		 6376
6377 /* interrupts should cause a wake up from C3 */
3299254f 6378 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
dbb42748
VS		 6379
6380 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
6381 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
1038392b
VS		 6382
6383 I915_WRITE(MI_ARB_STATE,
6384 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
6f1d69b0
ED		 6385}
6386
1fa61106 6387static void i85x_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 6388{
6389 struct drm_i915_private *dev_priv = dev->dev_private;
6390
6391 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
54e472ae
VS		 6392
6393 /* interrupts should cause a wake up from C3 */
6394 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
6395 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
1038392b
VS		 6396
6397 I915_WRITE(MEM_MODE,
6398 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
6f1d69b0
ED		 6399}
6400
1fa61106 6401static void i830_init_clock_gating(struct drm_device *dev)
6f1d69b0
ED		 6402{
6403 struct drm_i915_private *dev_priv = dev->dev_private;
6404
6405 I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
1038392b
VS		 6406
6407 I915_WRITE(MEM_MODE,
6408 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
6409 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
6f1d69b0
ED		 6410}
6411
6f1d69b0
ED		 6412void intel_init_clock_gating(struct drm_device *dev)
6413{
6414 struct drm_i915_private *dev_priv = dev->dev_private;
6415
c57e3551
DL		 6416 if (dev_priv->display.init_clock_gating)
6417 dev_priv->display.init_clock_gating(dev);
6f1d69b0
ED		 6418}
6419
7d708ee4
ID		 6420void intel_suspend_hw(struct drm_device *dev)
6421{
6422 if (HAS_PCH_LPT(dev))
6423 lpt_suspend_hw(dev);
6424}
6425
1fa61106
ED		 6426/* Set up chip specific power management-related functions */
6427void intel_init_pm(struct drm_device *dev)
6428{
6429 struct drm_i915_private *dev_priv = dev->dev_private;
6430
7ff0ebcc 6431 intel_fbc_init(dev_priv);
1fa61106 6432
c921aba8
DV		 6433 /* For cxsr */
6434 if (IS_PINEVIEW(dev))
6435 i915_pineview_get_mem_freq(dev);
6436 else if (IS_GEN5(dev))
6437 i915_ironlake_get_mem_freq(dev);
6438
1fa61106 6439 /* For FIFO watermark updates */
f5ed50cb 6440 if (INTEL_INFO(dev)->gen >= 9) {
2af30a5c
PB		 6441 skl_setup_wm_latency(dev);
6442
45db2194 6443 dev_priv->display.init_clock_gating = skl_init_clock_gating;
2d41c0b5
PB		 6444 dev_priv->display.update_wm = skl_update_wm;
6445 dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
c83155a6 6446 } else if (HAS_PCH_SPLIT(dev)) {
fa50ad61 6447 ilk_setup_wm_latency(dev);
53615a5e 6448
bd602544
VS		 6449 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
6450 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
6451 (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
6452 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
6453 dev_priv->display.update_wm = ilk_update_wm;
6454 dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
6455 } else {
6456 DRM_DEBUG_KMS("Failed to read display plane latency. "
6457 "Disable CxSR\n");
6458 }
6459
6460 if (IS_GEN5(dev))
1fa61106 6461 dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
bd602544 6462 else if (IS_GEN6(dev))
1fa61106 6463 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
bd602544 6464 else if (IS_IVYBRIDGE(dev))
1fa61106 6465 dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
bd602544 6466 else if (IS_HASWELL(dev))
cad2a2d7 6467 dev_priv->display.init_clock_gating = haswell_init_clock_gating;
bd602544 6468 else if (INTEL_INFO(dev)->gen == 8)
47c2bd97 6469 dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
a4565da8 6470 } else if (IS_CHERRYVIEW(dev)) {
ae80152d 6471 dev_priv->display.update_wm = valleyview_update_wm;
01e184cc 6472 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
a4565da8
VS		 6473 dev_priv->display.init_clock_gating =
6474 cherryview_init_clock_gating;
1fa61106
ED		 6475 } else if (IS_VALLEYVIEW(dev)) {
6476 dev_priv->display.update_wm = valleyview_update_wm;
01e184cc 6477 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
1fa61106
ED		 6478 dev_priv->display.init_clock_gating =
6479 valleyview_init_clock_gating;
1fa61106
ED		 6480 } else if (IS_PINEVIEW(dev)) {
6481 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
6482 dev_priv->is_ddr3,
6483 dev_priv->fsb_freq,
6484 dev_priv->mem_freq)) {
6485 DRM_INFO("failed to find known CxSR latency "
6486 "(found ddr%s fsb freq %d, mem freq %d), "
6487 "disabling CxSR\n",
6488 (dev_priv->is_ddr3 == 1) ? "3" : "2",
6489 dev_priv->fsb_freq, dev_priv->mem_freq);
6490 /* Disable CxSR and never update its watermark again */
5209b1f4 6491 intel_set_memory_cxsr(dev_priv, false);
1fa61106
ED		 6492 dev_priv->display.update_wm = NULL;
6493 } else
6494 dev_priv->display.update_wm = pineview_update_wm;
6495 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
6496 } else if (IS_G4X(dev)) {
6497 dev_priv->display.update_wm = g4x_update_wm;
6498 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
6499 } else if (IS_GEN4(dev)) {
6500 dev_priv->display.update_wm = i965_update_wm;
6501 if (IS_CRESTLINE(dev))
6502 dev_priv->display.init_clock_gating = crestline_init_clock_gating;
6503 else if (IS_BROADWATER(dev))
6504 dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
6505 } else if (IS_GEN3(dev)) {
6506 dev_priv->display.update_wm = i9xx_update_wm;
6507 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
6508 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
feb56b93
DV		 6509 } else if (IS_GEN2(dev)) {
6510 if (INTEL_INFO(dev)->num_pipes == 1) {
6511 dev_priv->display.update_wm = i845_update_wm;
1fa61106 6512 dev_priv->display.get_fifo_size = i845_get_fifo_size;
feb56b93
DV		 6513 } else {
6514 dev_priv->display.update_wm = i9xx_update_wm;
1fa61106 6515 dev_priv->display.get_fifo_size = i830_get_fifo_size;
feb56b93
DV		 6516 }
6517
6518 if (IS_I85X(dev) || IS_I865G(dev))
6519 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
6520 else
6521 dev_priv->display.init_clock_gating = i830_init_clock_gating;
6522 } else {
6523 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
1fa61106
ED		 6524 }
6525}
6526
151a49d0 6527int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
42c0526c 6528{
4fc688ce 6529 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
42c0526c
BW		 6530
6531 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6532 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
6533 return -EAGAIN;
6534 }
6535
6536 I915_WRITE(GEN6_PCODE_DATA, *val);
dddab346 6537 I915_WRITE(GEN6_PCODE_DATA1, 0);
42c0526c
BW		 6538 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6539
6540 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6541 500)) {
6542 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
6543 return -ETIMEDOUT;
6544 }
6545
6546 *val = I915_READ(GEN6_PCODE_DATA);
6547 I915_WRITE(GEN6_PCODE_DATA, 0);
6548
6549 return 0;
6550}
6551
151a49d0 6552int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
42c0526c 6553{
4fc688ce 6554 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
42c0526c
BW		 6555
6556 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6557 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
6558 return -EAGAIN;
6559 }
6560
6561 I915_WRITE(GEN6_PCODE_DATA, val);
6562 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6563
6564 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6565 500)) {
6566 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
6567 return -ETIMEDOUT;
6568 }
6569
6570 I915_WRITE(GEN6_PCODE_DATA, 0);
6571
6572 return 0;
6573}
a0e4e199 6574
dd06f88c 6575static int vlv_gpu_freq_div(unsigned int czclk_freq)
855ba3be 6576{
dd06f88c
VS		 6577 switch (czclk_freq) {
6578 case 200:
6579 return 10;
6580 case 267:
6581 return 12;
6582 case 320:
6583 case 333:
dd06f88c 6584 return 16;
ab3fb157
VS		 6585 case 400:
6586 return 20;
855ba3be
JB		 6587 default:
6588 return -1;
6589 }
dd06f88c 6590}
855ba3be 6591
dd06f88c
VS		 6592static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
6593{
6594 int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
6595
6596 div = vlv_gpu_freq_div(czclk_freq);
6597 if (div < 0)
6598 return div;
6599
6600 return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div);
855ba3be
JB		 6601}
6602
b55dd647 6603static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
855ba3be 6604{
dd06f88c 6605 int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
855ba3be 6606
dd06f88c
VS		 6607 mul = vlv_gpu_freq_div(czclk_freq);
6608 if (mul < 0)
6609 return mul;
855ba3be 6610
dd06f88c 6611 return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6;
855ba3be
JB		 6612}
6613
b55dd647 6614static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
22b1b2f8 6615{
dd06f88c 6616 int div, czclk_freq = dev_priv->rps.cz_freq;
22b1b2f8 6617
dd06f88c
VS		 6618 div = vlv_gpu_freq_div(czclk_freq) / 2;
6619 if (div < 0)
6620 return div;
22b1b2f8 6621
dd06f88c 6622 return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2;
22b1b2f8
D		 6623}
6624
b55dd647 6625static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
22b1b2f8 6626{
dd06f88c 6627 int mul, czclk_freq = dev_priv->rps.cz_freq;
22b1b2f8 6628
dd06f88c
VS		 6629 mul = vlv_gpu_freq_div(czclk_freq) / 2;
6630 if (mul < 0)
6631 return mul;
22b1b2f8 6632
1c14762d 6633 /* CHV needs even values */
dd06f88c 6634 return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2;
22b1b2f8
D		 6635}
6636
616bc820 6637int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
22b1b2f8 6638{
22b1b2f8 6639 if (IS_CHERRYVIEW(dev_priv->dev))
616bc820 6640 return chv_gpu_freq(dev_priv, val);
22b1b2f8 6641 else if (IS_VALLEYVIEW(dev_priv->dev))
616bc820
VS		 6642 return byt_gpu_freq(dev_priv, val);
6643 else
6644 return val * GT_FREQUENCY_MULTIPLIER;
22b1b2f8
D		 6645}
6646
616bc820
VS		 6647int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
6648{
22b1b2f8 6649 if (IS_CHERRYVIEW(dev_priv->dev))
616bc820 6650 return chv_freq_opcode(dev_priv, val);
22b1b2f8 6651 else if (IS_VALLEYVIEW(dev_priv->dev))
616bc820
VS		 6652 return byt_freq_opcode(dev_priv, val);
6653 else
6654 return val / GT_FREQUENCY_MULTIPLIER;
6655}
22b1b2f8 6656
f742a552 6657void intel_pm_setup(struct drm_device *dev)
907b28c5
CW		 6658{
6659 struct drm_i915_private *dev_priv = dev->dev_private;
6660
f742a552
DV		 6661 mutex_init(&dev_priv->rps.hw_lock);
6662
907b28c5
CW		 6663 INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
6664 intel_gen6_powersave_work);
5d584b2e 6665
33688d95 6666 dev_priv->pm.suspended = false;
907b28c5 6667}
ubuntu-bionic-kernel mirror