]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/gpu/drm/i915/intel_runtime_pm.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/jikos/livep...
[mirror_ubuntu-artful-kernel.git] / drivers / gpu / drm / i915 / intel_runtime_pm.c
CommitLineData
9c065a7d
DV		 1/*
2 * Copyright © 2012-2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 * Daniel Vetter <daniel.vetter@ffwll.ch>
26 *
27 */
28
29#include <linux/pm_runtime.h>
30#include <linux/vgaarb.h>
31
32#include "i915_drv.h"
33#include "intel_drv.h"
9c065a7d 34
e4e7684f
DV		 35/**
36 * DOC: runtime pm
37 *
38 * The i915 driver supports dynamic enabling and disabling of entire hardware
39 * blocks at runtime. This is especially important on the display side where
40 * software is supposed to control many power gates manually on recent hardware,
41 * since on the GT side a lot of the power management is done by the hardware.
42 * But even there some manual control at the device level is required.
43 *
44 * Since i915 supports a diverse set of platforms with a unified codebase and
45 * hardware engineers just love to shuffle functionality around between power
46 * domains there's a sizeable amount of indirection required. This file provides
47 * generic functions to the driver for grabbing and releasing references for
48 * abstract power domains. It then maps those to the actual power wells
49 * present for a given platform.
50 */
51
5aefb239
SS		 52bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
53 int power_well_id);
54
9c8d0b8e
ID		 55static struct i915_power_well *
56lookup_power_well(struct drm_i915_private *dev_priv, int power_well_id);
57
9895ad03
DS		 58const char *
59intel_display_power_domain_str(enum intel_display_power_domain domain)
60{
61 switch (domain) {
62 case POWER_DOMAIN_PIPE_A:
63 return "PIPE_A";
64 case POWER_DOMAIN_PIPE_B:
65 return "PIPE_B";
66 case POWER_DOMAIN_PIPE_C:
67 return "PIPE_C";
68 case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
69 return "PIPE_A_PANEL_FITTER";
70 case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
71 return "PIPE_B_PANEL_FITTER";
72 case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
73 return "PIPE_C_PANEL_FITTER";
74 case POWER_DOMAIN_TRANSCODER_A:
75 return "TRANSCODER_A";
76 case POWER_DOMAIN_TRANSCODER_B:
77 return "TRANSCODER_B";
78 case POWER_DOMAIN_TRANSCODER_C:
79 return "TRANSCODER_C";
80 case POWER_DOMAIN_TRANSCODER_EDP:
81 return "TRANSCODER_EDP";
4d1de975
JN		 82 case POWER_DOMAIN_TRANSCODER_DSI_A:
83 return "TRANSCODER_DSI_A";
84 case POWER_DOMAIN_TRANSCODER_DSI_C:
85 return "TRANSCODER_DSI_C";
9895ad03
DS		 86 case POWER_DOMAIN_PORT_DDI_A_LANES:
87 return "PORT_DDI_A_LANES";
88 case POWER_DOMAIN_PORT_DDI_B_LANES:
89 return "PORT_DDI_B_LANES";
90 case POWER_DOMAIN_PORT_DDI_C_LANES:
91 return "PORT_DDI_C_LANES";
92 case POWER_DOMAIN_PORT_DDI_D_LANES:
93 return "PORT_DDI_D_LANES";
94 case POWER_DOMAIN_PORT_DDI_E_LANES:
95 return "PORT_DDI_E_LANES";
62b69566
ACO		 96 case POWER_DOMAIN_PORT_DDI_A_IO:
97 return "PORT_DDI_A_IO";
98 case POWER_DOMAIN_PORT_DDI_B_IO:
99 return "PORT_DDI_B_IO";
100 case POWER_DOMAIN_PORT_DDI_C_IO:
101 return "PORT_DDI_C_IO";
102 case POWER_DOMAIN_PORT_DDI_D_IO:
103 return "PORT_DDI_D_IO";
104 case POWER_DOMAIN_PORT_DDI_E_IO:
105 return "PORT_DDI_E_IO";
9895ad03
DS		 106 case POWER_DOMAIN_PORT_DSI:
107 return "PORT_DSI";
108 case POWER_DOMAIN_PORT_CRT:
109 return "PORT_CRT";
110 case POWER_DOMAIN_PORT_OTHER:
111 return "PORT_OTHER";
112 case POWER_DOMAIN_VGA:
113 return "VGA";
114 case POWER_DOMAIN_AUDIO:
115 return "AUDIO";
116 case POWER_DOMAIN_PLLS:
117 return "PLLS";
118 case POWER_DOMAIN_AUX_A:
119 return "AUX_A";
120 case POWER_DOMAIN_AUX_B:
121 return "AUX_B";
122 case POWER_DOMAIN_AUX_C:
123 return "AUX_C";
124 case POWER_DOMAIN_AUX_D:
125 return "AUX_D";
126 case POWER_DOMAIN_GMBUS:
127 return "GMBUS";
128 case POWER_DOMAIN_INIT:
129 return "INIT";
130 case POWER_DOMAIN_MODESET:
131 return "MODESET";
132 default:
133 MISSING_CASE(domain);
134 return "?";
135 }
136}
137
e8ca9320
DL		 138static void intel_power_well_enable(struct drm_i915_private *dev_priv,
139 struct i915_power_well *power_well)
140{
141 DRM_DEBUG_KMS("enabling %s\n", power_well->name);
142 power_well->ops->enable(dev_priv, power_well);
143 power_well->hw_enabled = true;
144}
145
dcddab3a
DL		 146static void intel_power_well_disable(struct drm_i915_private *dev_priv,
147 struct i915_power_well *power_well)
148{
149 DRM_DEBUG_KMS("disabling %s\n", power_well->name);
150 power_well->hw_enabled = false;
151 power_well->ops->disable(dev_priv, power_well);
152}
153
b409ca95
ID		 154static void intel_power_well_get(struct drm_i915_private *dev_priv,
155 struct i915_power_well *power_well)
156{
157 if (!power_well->count++)
158 intel_power_well_enable(dev_priv, power_well);
159}
160
161static void intel_power_well_put(struct drm_i915_private *dev_priv,
162 struct i915_power_well *power_well)
163{
164 WARN(!power_well->count, "Use count on power well %s is already zero",
165 power_well->name);
166
167 if (!--power_well->count)
168 intel_power_well_disable(dev_priv, power_well);
169}
170
e4e7684f 171/*
9c065a7d
DV		 172 * We should only use the power well if we explicitly asked the hardware to
173 * enable it, so check if it's enabled and also check if we've requested it to
174 * be enabled.
175 */
176static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
177 struct i915_power_well *power_well)
178{
179 return I915_READ(HSW_PWR_WELL_DRIVER) ==
180 (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED);
181}
182
e4e7684f
DV		 183/**
184 * __intel_display_power_is_enabled - unlocked check for a power domain
185 * @dev_priv: i915 device instance
186 * @domain: power domain to check
187 *
188 * This is the unlocked version of intel_display_power_is_enabled() and should
189 * only be used from error capture and recovery code where deadlocks are
190 * possible.
191 *
192 * Returns:
193 * True when the power domain is enabled, false otherwise.
194 */
f458ebbc
DV		 195bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
196 enum intel_display_power_domain domain)
9c065a7d 197{
9c065a7d
DV		 198 struct i915_power_well *power_well;
199 bool is_enabled;
9c065a7d
DV		 200
201 if (dev_priv->pm.suspended)
202 return false;
203
9c065a7d
DV		 204 is_enabled = true;
205
75ccb2ec 206 for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain)) {
9c065a7d
DV		 207 if (power_well->always_on)
208 continue;
209
210 if (!power_well->hw_enabled) {
211 is_enabled = false;
212 break;
213 }
214 }
215
216 return is_enabled;
217}
218
e4e7684f 219/**
f61ccae3 220 * intel_display_power_is_enabled - check for a power domain
e4e7684f
DV		 221 * @dev_priv: i915 device instance
222 * @domain: power domain to check
223 *
224 * This function can be used to check the hw power domain state. It is mostly
225 * used in hardware state readout functions. Everywhere else code should rely
226 * upon explicit power domain reference counting to ensure that the hardware
227 * block is powered up before accessing it.
228 *
229 * Callers must hold the relevant modesetting locks to ensure that concurrent
230 * threads can't disable the power well while the caller tries to read a few
231 * registers.
232 *
233 * Returns:
234 * True when the power domain is enabled, false otherwise.
235 */
f458ebbc
DV		 236bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
237 enum intel_display_power_domain domain)
9c065a7d
DV		 238{
239 struct i915_power_domains *power_domains;
240 bool ret;
241
242 power_domains = &dev_priv->power_domains;
243
244 mutex_lock(&power_domains->lock);
f458ebbc 245 ret = __intel_display_power_is_enabled(dev_priv, domain);
9c065a7d
DV		 246 mutex_unlock(&power_domains->lock);
247
248 return ret;
249}
250
e4e7684f
DV		 251/**
252 * intel_display_set_init_power - set the initial power domain state
253 * @dev_priv: i915 device instance
254 * @enable: whether to enable or disable the initial power domain state
255 *
256 * For simplicity our driver load/unload and system suspend/resume code assumes
257 * that all power domains are always enabled. This functions controls the state
258 * of this little hack. While the initial power domain state is enabled runtime
259 * pm is effectively disabled.
260 */
d9bc89d9
DV		 261void intel_display_set_init_power(struct drm_i915_private *dev_priv,
262 bool enable)
263{
264 if (dev_priv->power_domains.init_power_on == enable)
265 return;
266
267 if (enable)
268 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
269 else
270 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
271
272 dev_priv->power_domains.init_power_on = enable;
273}
274
9c065a7d
DV		 275/*
276 * Starting with Haswell, we have a "Power Down Well" that can be turned off
277 * when not needed anymore. We have 4 registers that can request the power well
278 * to be enabled, and it will only be disabled if none of the registers is
279 * requesting it to be enabled.
280 */
281static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
282{
52a05c30 283 struct pci_dev *pdev = dev_priv->drm.pdev;
9c065a7d
DV		 284
285 /*
286 * After we re-enable the power well, if we touch VGA register 0x3d5
287 * we'll get unclaimed register interrupts. This stops after we write
288 * anything to the VGA MSR register. The vgacon module uses this
289 * register all the time, so if we unbind our driver and, as a
290 * consequence, bind vgacon, we'll get stuck in an infinite loop at
291 * console_unlock(). So make here we touch the VGA MSR register, making
292 * sure vgacon can keep working normally without triggering interrupts
293 * and error messages.
294 */
52a05c30 295 vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
9c065a7d 296 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
52a05c30 297 vga_put(pdev, VGA_RSRC_LEGACY_IO);
9c065a7d 298
8652744b 299 if (IS_BROADWELL(dev_priv))
4c6c03be
DL		 300 gen8_irq_power_well_post_enable(dev_priv,
301 1 << PIPE_C | 1 << PIPE_B);
9c065a7d
DV		 302}
303
aae8ba84
VS		 304static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv)
305{
306 if (IS_BROADWELL(dev_priv))
307 gen8_irq_power_well_pre_disable(dev_priv,
308 1 << PIPE_C | 1 << PIPE_B);
309}
310
d14c0343
DL		 311static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
312 struct i915_power_well *power_well)
313{
52a05c30 314 struct pci_dev *pdev = dev_priv->drm.pdev;
d14c0343
DL		 315
316 /*
317 * After we re-enable the power well, if we touch VGA register 0x3d5
318 * we'll get unclaimed register interrupts. This stops after we write
319 * anything to the VGA MSR register. The vgacon module uses this
320 * register all the time, so if we unbind our driver and, as a
321 * consequence, bind vgacon, we'll get stuck in an infinite loop at
322 * console_unlock(). So make here we touch the VGA MSR register, making
323 * sure vgacon can keep working normally without triggering interrupts
324 * and error messages.
325 */
01c3faa7 326 if (power_well->id == SKL_DISP_PW_2) {
52a05c30 327 vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
d14c0343 328 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
52a05c30 329 vga_put(pdev, VGA_RSRC_LEGACY_IO);
d14c0343
DL		 330
331 gen8_irq_power_well_post_enable(dev_priv,
332 1 << PIPE_C | 1 << PIPE_B);
333 }
d14c0343
DL		 334}
335
aae8ba84
VS		 336static void skl_power_well_pre_disable(struct drm_i915_private *dev_priv,
337 struct i915_power_well *power_well)
338{
01c3faa7 339 if (power_well->id == SKL_DISP_PW_2)
aae8ba84
VS		 340 gen8_irq_power_well_pre_disable(dev_priv,
341 1 << PIPE_C | 1 << PIPE_B);
342}
343
9c065a7d
DV		 344static void hsw_set_power_well(struct drm_i915_private *dev_priv,
345 struct i915_power_well *power_well, bool enable)
346{
347 bool is_enabled, enable_requested;
348 uint32_t tmp;
349
350 tmp = I915_READ(HSW_PWR_WELL_DRIVER);
351 is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
352 enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
353
354 if (enable) {
355 if (!enable_requested)
356 I915_WRITE(HSW_PWR_WELL_DRIVER,
357 HSW_PWR_WELL_ENABLE_REQUEST);
358
359 if (!is_enabled) {
360 DRM_DEBUG_KMS("Enabling power well\n");
2c2ccc3a
CW		 361 if (intel_wait_for_register(dev_priv,
362 HSW_PWR_WELL_DRIVER,
363 HSW_PWR_WELL_STATE_ENABLED,
364 HSW_PWR_WELL_STATE_ENABLED,
365 20))
9c065a7d 366 DRM_ERROR("Timeout enabling power well\n");
6d729bff 367 hsw_power_well_post_enable(dev_priv);
9c065a7d
DV		 368 }
369
9c065a7d
DV		 370 } else {
371 if (enable_requested) {
aae8ba84 372 hsw_power_well_pre_disable(dev_priv);
9c065a7d
DV		 373 I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
374 POSTING_READ(HSW_PWR_WELL_DRIVER);
375 DRM_DEBUG_KMS("Requesting to disable the power well\n");
376 }
377 }
378}
379
94dd5138 380#define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS ( \
d8fc70b7
ACO		 381 BIT_ULL(POWER_DOMAIN_TRANSCODER_A) | \
382 BIT_ULL(POWER_DOMAIN_PIPE_B) | \
383 BIT_ULL(POWER_DOMAIN_TRANSCODER_B) | \
384 BIT_ULL(POWER_DOMAIN_PIPE_C) | \
385 BIT_ULL(POWER_DOMAIN_TRANSCODER_C) | \
386 BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \
387 BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) | \
388 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
389 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
390 BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) | \
391 BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) | \
392 BIT_ULL(POWER_DOMAIN_AUX_B) | \
393 BIT_ULL(POWER_DOMAIN_AUX_C) | \
394 BIT_ULL(POWER_DOMAIN_AUX_D) | \
395 BIT_ULL(POWER_DOMAIN_AUDIO) | \
396 BIT_ULL(POWER_DOMAIN_VGA) | \
397 BIT_ULL(POWER_DOMAIN_INIT))
62b69566
ACO		 398#define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS ( \
399 BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) | \
400 BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) | \
d8fc70b7 401 BIT_ULL(POWER_DOMAIN_INIT))
62b69566
ACO		 402#define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS ( \
403 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) | \
d8fc70b7 404 BIT_ULL(POWER_DOMAIN_INIT))
62b69566
ACO		 405#define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS ( \
406 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) | \
d8fc70b7 407 BIT_ULL(POWER_DOMAIN_INIT))
62b69566
ACO		 408#define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS ( \
409 BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) | \
d8fc70b7 410 BIT_ULL(POWER_DOMAIN_INIT))
9f836f90
PJ		 411#define SKL_DISPLAY_DC_OFF_POWER_DOMAINS ( \
412 SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS | \
d8fc70b7
ACO		 413 BIT_ULL(POWER_DOMAIN_MODESET) | \
414 BIT_ULL(POWER_DOMAIN_AUX_A) | \
415 BIT_ULL(POWER_DOMAIN_INIT))
94dd5138 416
0b4a2a36 417#define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS ( \
d8fc70b7
ACO		 418 BIT_ULL(POWER_DOMAIN_TRANSCODER_A) | \
419 BIT_ULL(POWER_DOMAIN_PIPE_B) | \
420 BIT_ULL(POWER_DOMAIN_TRANSCODER_B) | \
421 BIT_ULL(POWER_DOMAIN_PIPE_C) | \
422 BIT_ULL(POWER_DOMAIN_TRANSCODER_C) | \
423 BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \
424 BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) | \
425 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
426 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
427 BIT_ULL(POWER_DOMAIN_AUX_B) | \
428 BIT_ULL(POWER_DOMAIN_AUX_C) | \
429 BIT_ULL(POWER_DOMAIN_AUDIO) | \
430 BIT_ULL(POWER_DOMAIN_VGA) | \
431 BIT_ULL(POWER_DOMAIN_GMBUS) | \
432 BIT_ULL(POWER_DOMAIN_INIT))
9f836f90
PJ		 433#define BXT_DISPLAY_DC_OFF_POWER_DOMAINS ( \
434 BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS | \
d8fc70b7
ACO		 435 BIT_ULL(POWER_DOMAIN_MODESET) | \
436 BIT_ULL(POWER_DOMAIN_AUX_A) | \
437 BIT_ULL(POWER_DOMAIN_INIT))
9c8d0b8e 438#define BXT_DPIO_CMN_A_POWER_DOMAINS ( \
d8fc70b7
ACO		 439 BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) | \
440 BIT_ULL(POWER_DOMAIN_AUX_A) | \
441 BIT_ULL(POWER_DOMAIN_INIT))
9c8d0b8e 442#define BXT_DPIO_CMN_BC_POWER_DOMAINS ( \
d8fc70b7
ACO		 443 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
444 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
445 BIT_ULL(POWER_DOMAIN_AUX_B) | \
446 BIT_ULL(POWER_DOMAIN_AUX_C) | \
447 BIT_ULL(POWER_DOMAIN_INIT))
0b4a2a36 448
0d03926d 449#define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS ( \
d8fc70b7
ACO		 450 BIT_ULL(POWER_DOMAIN_TRANSCODER_A) | \
451 BIT_ULL(POWER_DOMAIN_PIPE_B) | \
452 BIT_ULL(POWER_DOMAIN_TRANSCODER_B) | \
453 BIT_ULL(POWER_DOMAIN_PIPE_C) | \
454 BIT_ULL(POWER_DOMAIN_TRANSCODER_C) | \
455 BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \
456 BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) | \
457 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
458 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
459 BIT_ULL(POWER_DOMAIN_AUX_B) | \
460 BIT_ULL(POWER_DOMAIN_AUX_C) | \
461 BIT_ULL(POWER_DOMAIN_AUDIO) | \
462 BIT_ULL(POWER_DOMAIN_VGA) | \
463 BIT_ULL(POWER_DOMAIN_INIT))
62b69566
ACO		 464#define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS ( \
465 BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
466#define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS ( \
467 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
468#define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS ( \
469 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
0a116ce8 470#define GLK_DPIO_CMN_A_POWER_DOMAINS ( \
d8fc70b7
ACO		 471 BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) | \
472 BIT_ULL(POWER_DOMAIN_AUX_A) | \
473 BIT_ULL(POWER_DOMAIN_INIT))
0a116ce8 474#define GLK_DPIO_CMN_B_POWER_DOMAINS ( \
d8fc70b7
ACO		 475 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
476 BIT_ULL(POWER_DOMAIN_AUX_B) | \
477 BIT_ULL(POWER_DOMAIN_INIT))
0a116ce8 478#define GLK_DPIO_CMN_C_POWER_DOMAINS ( \
d8fc70b7
ACO		 479 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
480 BIT_ULL(POWER_DOMAIN_AUX_C) | \
481 BIT_ULL(POWER_DOMAIN_INIT))
0d03926d 482#define GLK_DISPLAY_AUX_A_POWER_DOMAINS ( \
d8fc70b7
ACO		 483 BIT_ULL(POWER_DOMAIN_AUX_A) | \
484 BIT_ULL(POWER_DOMAIN_INIT))
0d03926d 485#define GLK_DISPLAY_AUX_B_POWER_DOMAINS ( \
d8fc70b7
ACO		 486 BIT_ULL(POWER_DOMAIN_AUX_B) | \
487 BIT_ULL(POWER_DOMAIN_INIT))
0d03926d 488#define GLK_DISPLAY_AUX_C_POWER_DOMAINS ( \
d8fc70b7
ACO		 489 BIT_ULL(POWER_DOMAIN_AUX_C) | \
490 BIT_ULL(POWER_DOMAIN_INIT))
0d03926d
ACO		 491#define GLK_DISPLAY_DC_OFF_POWER_DOMAINS ( \
492 GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS | \
d8fc70b7
ACO		 493 BIT_ULL(POWER_DOMAIN_MODESET) | \
494 BIT_ULL(POWER_DOMAIN_AUX_A) | \
495 BIT_ULL(POWER_DOMAIN_INIT))
0d03926d 496
664326f8
SK		 497static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
498{
bfcdabe8
ID		 499 WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
500 "DC9 already programmed to be enabled.\n");
501 WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
502 "DC5 still not disabled to enable DC9.\n");
503 WARN_ONCE(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on.\n");
504 WARN_ONCE(intel_irqs_enabled(dev_priv),
505 "Interrupts not disabled yet.\n");
664326f8
SK		 506
507 /*
508 * TODO: check for the following to verify the conditions to enter DC9
509 * state are satisfied:
510 * 1] Check relevant display engine registers to verify if mode set
511 * disable sequence was followed.
512 * 2] Check if display uninitialize sequence is initialized.
513 */
514}
515
516static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
517{
bfcdabe8
ID		 518 WARN_ONCE(intel_irqs_enabled(dev_priv),
519 "Interrupts not disabled yet.\n");
520 WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
521 "DC5 still not disabled.\n");
664326f8
SK		 522
523 /*
524 * TODO: check for the following to verify DC9 state was indeed
525 * entered before programming to disable it:
526 * 1] Check relevant display engine registers to verify if mode
527 * set disable sequence was followed.
528 * 2] Check if display uninitialize sequence is initialized.
529 */
530}
531
779cb5d3
MK		 532static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
533 u32 state)
534{
535 int rewrites = 0;
536 int rereads = 0;
537 u32 v;
538
539 I915_WRITE(DC_STATE_EN, state);
540
541 /* It has been observed that disabling the dc6 state sometimes
542 * doesn't stick and dmc keeps returning old value. Make sure
543 * the write really sticks enough times and also force rewrite until
544 * we are confident that state is exactly what we want.
545 */
546 do {
547 v = I915_READ(DC_STATE_EN);
548
549 if (v != state) {
550 I915_WRITE(DC_STATE_EN, state);
551 rewrites++;
552 rereads = 0;
553 } else if (rereads++ > 5) {
554 break;
555 }
556
557 } while (rewrites < 100);
558
559 if (v != state)
560 DRM_ERROR("Writing dc state to 0x%x failed, now 0x%x\n",
561 state, v);
562
563 /* Most of the times we need one retry, avoid spam */
564 if (rewrites > 1)
565 DRM_DEBUG_KMS("Rewrote dc state to 0x%x %d times\n",
566 state, rewrites);
567}
568
da2f41d1 569static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
664326f8 570{
da2f41d1 571 u32 mask;
664326f8 572
13ae3a0d 573 mask = DC_STATE_EN_UPTO_DC5;
cc3f90f0 574 if (IS_GEN9_LP(dev_priv))
13ae3a0d
ID		 575 mask |= DC_STATE_EN_DC9;
576 else
577 mask |= DC_STATE_EN_UPTO_DC6;
664326f8 578
da2f41d1
ID		 579 return mask;
580}
581
582void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
583{
584 u32 val;
585
586 val = I915_READ(DC_STATE_EN) & gen9_dc_mask(dev_priv);
587
588 DRM_DEBUG_KMS("Resetting DC state tracking from %02x to %02x\n",
589 dev_priv->csr.dc_state, val);
590 dev_priv->csr.dc_state = val;
591}
592
593static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
594{
595 uint32_t val;
596 uint32_t mask;
597
a37baf3b
ID		 598 if (WARN_ON_ONCE(state & ~dev_priv->csr.allowed_dc_mask))
599 state &= dev_priv->csr.allowed_dc_mask;
443646c7 600
664326f8 601 val = I915_READ(DC_STATE_EN);
da2f41d1 602 mask = gen9_dc_mask(dev_priv);
13ae3a0d
ID		 603 DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n",
604 val & mask, state);
832dba88
PJ		 605
606 /* Check if DMC is ignoring our DC state requests */
607 if ((val & mask) != dev_priv->csr.dc_state)
608 DRM_ERROR("DC state mismatch (0x%x -> 0x%x)\n",
609 dev_priv->csr.dc_state, val & mask);
610
13ae3a0d
ID		 611 val &= ~mask;
612 val |= state;
779cb5d3
MK		 613
614 gen9_write_dc_state(dev_priv, val);
832dba88
PJ		 615
616 dev_priv->csr.dc_state = val & mask;
664326f8
SK		 617}
618
13ae3a0d 619void bxt_enable_dc9(struct drm_i915_private *dev_priv)
664326f8 620{
13ae3a0d
ID		 621 assert_can_enable_dc9(dev_priv);
622
623 DRM_DEBUG_KMS("Enabling DC9\n");
664326f8 624
78597996 625 intel_power_sequencer_reset(dev_priv);
13ae3a0d
ID		 626 gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
627}
628
629void bxt_disable_dc9(struct drm_i915_private *dev_priv)
630{
664326f8
SK		 631 assert_can_disable_dc9(dev_priv);
632
633 DRM_DEBUG_KMS("Disabling DC9\n");
634
13ae3a0d 635 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
8090ba8c
ID		 636
637 intel_pps_unlock_regs_wa(dev_priv);
664326f8
SK		 638}
639
af5fead2
DV		 640static void assert_csr_loaded(struct drm_i915_private *dev_priv)
641{
642 WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
643 "CSR program storage start is NULL\n");
644 WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
645 WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
646}
647
5aefb239 648static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
dc174300 649{
5aefb239
SS		 650 bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
651 SKL_DISP_PW_2);
652
6ff8ab0d 653 WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n");
5aefb239 654
6ff8ab0d
JB		 655 WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
656 "DC5 already programmed to be enabled.\n");
c9b8846a 657 assert_rpm_wakelock_held(dev_priv);
5aefb239
SS		 658
659 assert_csr_loaded(dev_priv);
660}
661
f62c79b3 662void gen9_enable_dc5(struct drm_i915_private *dev_priv)
5aefb239 663{
5aefb239 664 assert_can_enable_dc5(dev_priv);
6b457d31
SK		 665
666 DRM_DEBUG_KMS("Enabling DC5\n");
667
13ae3a0d 668 gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
dc174300
SS		 669}
670
93c7cb6c 671static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
f75a1985 672{
6ff8ab0d
JB		 673 WARN_ONCE(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
674 "Backlight is not disabled.\n");
675 WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
676 "DC6 already programmed to be enabled.\n");
93c7cb6c
SS		 677
678 assert_csr_loaded(dev_priv);
679}
680
0a9d2bed 681void skl_enable_dc6(struct drm_i915_private *dev_priv)
93c7cb6c 682{
93c7cb6c 683 assert_can_enable_dc6(dev_priv);
74b4f371
SK		 684
685 DRM_DEBUG_KMS("Enabling DC6\n");
686
13ae3a0d
ID		 687 gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
688
f75a1985
SS		 689}
690
0a9d2bed 691void skl_disable_dc6(struct drm_i915_private *dev_priv)
f75a1985 692{
74b4f371
SK		 693 DRM_DEBUG_KMS("Disabling DC6\n");
694
13ae3a0d 695 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
f75a1985
SS		 696}
697
c6782b76
ID		 698static void
699gen9_sanitize_power_well_requests(struct drm_i915_private *dev_priv,
700 struct i915_power_well *power_well)
701{
01c3faa7 702 enum skl_disp_power_wells power_well_id = power_well->id;
c6782b76
ID		 703 u32 val;
704 u32 mask;
705
706 mask = SKL_POWER_WELL_REQ(power_well_id);
707
708 val = I915_READ(HSW_PWR_WELL_KVMR);
709 if (WARN_ONCE(val & mask, "Clearing unexpected KVMR request for %s\n",
710 power_well->name))
711 I915_WRITE(HSW_PWR_WELL_KVMR, val & ~mask);
712
713 val = I915_READ(HSW_PWR_WELL_BIOS);
714 val |= I915_READ(HSW_PWR_WELL_DEBUG);
715
716 if (!(val & mask))
717 return;
718
719 /*
720 * DMC is known to force on the request bits for power well 1 on SKL
721 * and BXT and the misc IO power well on SKL but we don't expect any
722 * other request bits to be set, so WARN for those.
723 */
724 if (power_well_id == SKL_DISP_PW_1 ||
b976dc53 725 (IS_GEN9_BC(dev_priv) &&
80dbe997 726 power_well_id == SKL_DISP_PW_MISC_IO))
c6782b76
ID		 727 DRM_DEBUG_DRIVER("Clearing auxiliary requests for %s forced on "
728 "by DMC\n", power_well->name);
729 else
730 WARN_ONCE(1, "Clearing unexpected auxiliary requests for %s\n",
731 power_well->name);
732
733 I915_WRITE(HSW_PWR_WELL_BIOS, val & ~mask);
734 I915_WRITE(HSW_PWR_WELL_DEBUG, val & ~mask);
735}
736
94dd5138 737static void skl_set_power_well(struct drm_i915_private *dev_priv,
0d03926d 738 struct i915_power_well *power_well, bool enable)
94dd5138
S		 739{
740 uint32_t tmp, fuse_status;
741 uint32_t req_mask, state_mask;
2a51835f 742 bool is_enabled, enable_requested, check_fuse_status = false;
94dd5138
S		 743
744 tmp = I915_READ(HSW_PWR_WELL_DRIVER);
745 fuse_status = I915_READ(SKL_FUSE_STATUS);
746
01c3faa7 747 switch (power_well->id) {
94dd5138 748 case SKL_DISP_PW_1:
117c1148
CW		 749 if (intel_wait_for_register(dev_priv,
750 SKL_FUSE_STATUS,
751 SKL_FUSE_PG0_DIST_STATUS,
752 SKL_FUSE_PG0_DIST_STATUS,
753 1)) {
94dd5138
S		 754 DRM_ERROR("PG0 not enabled\n");
755 return;
756 }
757 break;
758 case SKL_DISP_PW_2:
759 if (!(fuse_status & SKL_FUSE_PG1_DIST_STATUS)) {
760 DRM_ERROR("PG1 in disabled state\n");
761 return;
762 }
763 break;
0d03926d
ACO		 764 case SKL_DISP_PW_MISC_IO:
765 case SKL_DISP_PW_DDI_A_E: /* GLK_DISP_PW_DDI_A */
94dd5138
S		 766 case SKL_DISP_PW_DDI_B:
767 case SKL_DISP_PW_DDI_C:
768 case SKL_DISP_PW_DDI_D:
0d03926d
ACO		 769 case GLK_DISP_PW_AUX_A:
770 case GLK_DISP_PW_AUX_B:
771 case GLK_DISP_PW_AUX_C:
94dd5138
S		 772 break;
773 default:
01c3faa7 774 WARN(1, "Unknown power well %lu\n", power_well->id);
94dd5138
S		 775 return;
776 }
777
01c3faa7 778 req_mask = SKL_POWER_WELL_REQ(power_well->id);
2a51835f 779 enable_requested = tmp & req_mask;
01c3faa7 780 state_mask = SKL_POWER_WELL_STATE(power_well->id);
2a51835f 781 is_enabled = tmp & state_mask;
94dd5138 782
aae8ba84
VS		 783 if (!enable && enable_requested)
784 skl_power_well_pre_disable(dev_priv, power_well);
785
94dd5138 786 if (enable) {
2a51835f 787 if (!enable_requested) {
dc174300
SS		 788 WARN((tmp & state_mask) &&
789 !I915_READ(HSW_PWR_WELL_BIOS),
790 "Invalid for power well status to be enabled, unless done by the BIOS, \
791 when request is to disable!\n");
94dd5138 792 I915_WRITE(HSW_PWR_WELL_DRIVER, tmp | req_mask);
94dd5138
S		 793 }
794
2a51835f 795 if (!is_enabled) {
510e6fdd 796 DRM_DEBUG_KMS("Enabling %s\n", power_well->name);
94dd5138
S		 797 check_fuse_status = true;
798 }
799 } else {
2a51835f 800 if (enable_requested) {
4a76f295
ID		 801 I915_WRITE(HSW_PWR_WELL_DRIVER, tmp & ~req_mask);
802 POSTING_READ(HSW_PWR_WELL_DRIVER);
803 DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
94dd5138 804 }
c6782b76 805
5f304c87 806 if (IS_GEN9(dev_priv))
c6782b76 807 gen9_sanitize_power_well_requests(dev_priv, power_well);
94dd5138
S		 808 }
809
1d963afa
ID		 810 if (wait_for(!!(I915_READ(HSW_PWR_WELL_DRIVER) & state_mask) == enable,
811 1))
812 DRM_ERROR("%s %s timeout\n",
813 power_well->name, enable ? "enable" : "disable");
814
94dd5138 815 if (check_fuse_status) {
01c3faa7 816 if (power_well->id == SKL_DISP_PW_1) {
8b00f55a
CW		 817 if (intel_wait_for_register(dev_priv,
818 SKL_FUSE_STATUS,
819 SKL_FUSE_PG1_DIST_STATUS,
820 SKL_FUSE_PG1_DIST_STATUS,
821 1))
94dd5138 822 DRM_ERROR("PG1 distributing status timeout\n");
01c3faa7 823 } else if (power_well->id == SKL_DISP_PW_2) {
8b00f55a
CW		 824 if (intel_wait_for_register(dev_priv,
825 SKL_FUSE_STATUS,
826 SKL_FUSE_PG2_DIST_STATUS,
827 SKL_FUSE_PG2_DIST_STATUS,
828 1))
94dd5138
S		 829 DRM_ERROR("PG2 distributing status timeout\n");
830 }
831 }
d14c0343
DL		 832
833 if (enable && !is_enabled)
834 skl_power_well_post_enable(dev_priv, power_well);
94dd5138
S		 835}
836
9c065a7d
DV		 837static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
838 struct i915_power_well *power_well)
839{
16e84914
ID		 840 /* Take over the request bit if set by BIOS. */
841 if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST) {
842 if (!(I915_READ(HSW_PWR_WELL_DRIVER) &
843 HSW_PWR_WELL_ENABLE_REQUEST))
844 I915_WRITE(HSW_PWR_WELL_DRIVER,
845 HSW_PWR_WELL_ENABLE_REQUEST);
9c065a7d 846 I915_WRITE(HSW_PWR_WELL_BIOS, 0);
16e84914 847 }
9c065a7d
DV		 848}
849
850static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
851 struct i915_power_well *power_well)
852{
853 hsw_set_power_well(dev_priv, power_well, true);
854}
855
856static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
857 struct i915_power_well *power_well)
858{
859 hsw_set_power_well(dev_priv, power_well, false);
860}
861
94dd5138
S		 862static bool skl_power_well_enabled(struct drm_i915_private *dev_priv,
863 struct i915_power_well *power_well)
864{
01c3faa7
ACO		 865 uint32_t mask = SKL_POWER_WELL_REQ(power_well->id) |
866 SKL_POWER_WELL_STATE(power_well->id);
94dd5138
S		 867
868 return (I915_READ(HSW_PWR_WELL_DRIVER) & mask) == mask;
869}
870
871static void skl_power_well_sync_hw(struct drm_i915_private *dev_priv,
872 struct i915_power_well *power_well)
873{
14544e13
ID		 874 uint32_t mask = SKL_POWER_WELL_REQ(power_well->id);
875 uint32_t bios_req = I915_READ(HSW_PWR_WELL_BIOS);
876
16e84914 877 /* Take over the request bit if set by BIOS. */
14544e13 878 if (bios_req & mask) {
16e84914
ID		 879 uint32_t drv_req = I915_READ(HSW_PWR_WELL_DRIVER);
880
881 if (!(drv_req & mask))
882 I915_WRITE(HSW_PWR_WELL_DRIVER, drv_req | mask);
14544e13
ID		 883 I915_WRITE(HSW_PWR_WELL_BIOS, bios_req & ~mask);
884 }
94dd5138
S		 885}
886
887static void skl_power_well_enable(struct drm_i915_private *dev_priv,
888 struct i915_power_well *power_well)
889{
890 skl_set_power_well(dev_priv, power_well, true);
891}
892
893static void skl_power_well_disable(struct drm_i915_private *dev_priv,
894 struct i915_power_well *power_well)
895{
896 skl_set_power_well(dev_priv, power_well, false);
897}
898
9c8d0b8e
ID		 899static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
900 struct i915_power_well *power_well)
901{
362624c9 902 bxt_ddi_phy_init(dev_priv, power_well->data);
9c8d0b8e
ID		 903}
904
905static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
906 struct i915_power_well *power_well)
907{
362624c9 908 bxt_ddi_phy_uninit(dev_priv, power_well->data);
9c8d0b8e
ID		 909}
910
911static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
912 struct i915_power_well *power_well)
913{
362624c9 914 return bxt_ddi_phy_is_enabled(dev_priv, power_well->data);
9c8d0b8e
ID		 915}
916
9c8d0b8e
ID		 917static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
918{
919 struct i915_power_well *power_well;
920
921 power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
922 if (power_well->count > 0)
362624c9 923 bxt_ddi_phy_verify_state(dev_priv, power_well->data);
9c8d0b8e
ID		 924
925 power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC);
926 if (power_well->count > 0)
362624c9 927 bxt_ddi_phy_verify_state(dev_priv, power_well->data);
0a116ce8
ACO		 928
929 if (IS_GEMINILAKE(dev_priv)) {
930 power_well = lookup_power_well(dev_priv, GLK_DPIO_CMN_C);
931 if (power_well->count > 0)
932 bxt_ddi_phy_verify_state(dev_priv, power_well->data);
933 }
9c8d0b8e
ID		 934}
935
9f836f90
PJ		 936static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
937 struct i915_power_well *power_well)
938{
939 return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
940}
941
18a8067c
VS		 942static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
943{
944 u32 tmp = I915_READ(DBUF_CTL);
945
946 WARN((tmp & (DBUF_POWER_STATE | DBUF_POWER_REQUEST)) !=
947 (DBUF_POWER_STATE | DBUF_POWER_REQUEST),
948 "Unexpected DBuf power power state (0x%08x)\n", tmp);
949}
950
9f836f90
PJ		 951static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
952 struct i915_power_well *power_well)
953{
49cd97a3
VS		 954 struct intel_cdclk_state cdclk_state = {};
955
5b773eb4 956 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
adc7f04b 957
49cd97a3
VS		 958 dev_priv->display.get_cdclk(dev_priv, &cdclk_state);
959 WARN_ON(!intel_cdclk_state_compare(&dev_priv->cdclk.hw, &cdclk_state));
342be926 960
18a8067c
VS		 961 gen9_assert_dbuf_enabled(dev_priv);
962
cc3f90f0 963 if (IS_GEN9_LP(dev_priv))
9c8d0b8e 964 bxt_verify_ddi_phy_power_wells(dev_priv);
9f836f90
PJ		 965}
966
967static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
968 struct i915_power_well *power_well)
969{
f74ed08d
ID		 970 if (!dev_priv->csr.dmc_payload)
971 return;
972
a37baf3b 973 if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
9f836f90 974 skl_enable_dc6(dev_priv);
a37baf3b 975 else if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
9f836f90
PJ		 976 gen9_enable_dc5(dev_priv);
977}
978
3c1b38e6
ID		 979static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
980 struct i915_power_well *power_well)
9f836f90 981{
9f836f90
PJ		 982}
983
9c065a7d
DV		 984static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
985 struct i915_power_well *power_well)
986{
987}
988
989static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
990 struct i915_power_well *power_well)
991{
992 return true;
993}
994
995static void vlv_set_power_well(struct drm_i915_private *dev_priv,
996 struct i915_power_well *power_well, bool enable)
997{
01c3faa7 998 enum punit_power_well power_well_id = power_well->id;
9c065a7d
DV		 999 u32 mask;
1000 u32 state;
1001 u32 ctrl;
1002
1003 mask = PUNIT_PWRGT_MASK(power_well_id);
1004 state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
1005 PUNIT_PWRGT_PWR_GATE(power_well_id);
1006
1007 mutex_lock(&dev_priv->rps.hw_lock);
1008
1009#define COND \
1010 ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
1011
1012 if (COND)
1013 goto out;
1014
1015 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
1016 ctrl &= ~mask;
1017 ctrl |= state;
1018 vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
1019
1020 if (wait_for(COND, 100))
7e35ab88 1021 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
9c065a7d
DV		 1022 state,
1023 vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
1024
1025#undef COND
1026
1027out:
1028 mutex_unlock(&dev_priv->rps.hw_lock);
1029}
1030
9c065a7d
DV		 1031static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
1032 struct i915_power_well *power_well)
1033{
1034 vlv_set_power_well(dev_priv, power_well, true);
1035}
1036
1037static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
1038 struct i915_power_well *power_well)
1039{
1040 vlv_set_power_well(dev_priv, power_well, false);
1041}
1042
1043static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
1044 struct i915_power_well *power_well)
1045{
01c3faa7 1046 int power_well_id = power_well->id;
9c065a7d
DV		 1047 bool enabled = false;
1048 u32 mask;
1049 u32 state;
1050 u32 ctrl;
1051
1052 mask = PUNIT_PWRGT_MASK(power_well_id);
1053 ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
1054
1055 mutex_lock(&dev_priv->rps.hw_lock);
1056
1057 state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
1058 /*
1059 * We only ever set the power-on and power-gate states, anything
1060 * else is unexpected.
1061 */
1062 WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
1063 state != PUNIT_PWRGT_PWR_GATE(power_well_id));
1064 if (state == ctrl)
1065 enabled = true;
1066
1067 /*
1068 * A transient state at this point would mean some unexpected party
1069 * is poking at the power controls too.
1070 */
1071 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
1072 WARN_ON(ctrl != state);
1073
1074 mutex_unlock(&dev_priv->rps.hw_lock);
1075
1076 return enabled;
1077}
1078
766078df
VS		 1079static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
1080{
721d4845
HG		 1081 u32 val;
1082
1083 /*
1084 * On driver load, a pipe may be active and driving a DSI display.
1085 * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
1086 * (and never recovering) in this case. intel_dsi_post_disable() will
1087 * clear it when we turn off the display.
1088 */
1089 val = I915_READ(DSPCLK_GATE_D);
1090 val &= DPOUNIT_CLOCK_GATE_DISABLE;
1091 val |= VRHUNIT_CLOCK_GATE_DISABLE;
1092 I915_WRITE(DSPCLK_GATE_D, val);
766078df
VS		 1093
1094 /*
1095 * Disable trickle feed and enable pnd deadline calculation
1096 */
1097 I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
1098 I915_WRITE(CBR1_VLV, 0);
19ab4ed3
VS		 1099
1100 WARN_ON(dev_priv->rawclk_freq == 0);
1101
1102 I915_WRITE(RAWCLK_FREQ_VLV,
1103 DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 1000));
766078df
VS		 1104}
1105
2be7d540 1106static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
9c065a7d 1107{
9504a892 1108 struct intel_encoder *encoder;
5a8fbb7d
VS		 1109 enum pipe pipe;
1110
1111 /*
1112 * Enable the CRI clock source so we can get at the
1113 * display and the reference clock for VGA
1114 * hotplug / manual detection. Supposedly DSI also
1115 * needs the ref clock up and running.
1116 *
1117 * CHV DPLL B/C have some issues if VGA mode is enabled.
1118 */
801388cb 1119 for_each_pipe(dev_priv, pipe) {
5a8fbb7d
VS		 1120 u32 val = I915_READ(DPLL(pipe));
1121
1122 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1123 if (pipe != PIPE_A)
1124 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1125
1126 I915_WRITE(DPLL(pipe), val);
1127 }
9c065a7d 1128
766078df
VS		 1129 vlv_init_display_clock_gating(dev_priv);
1130
9c065a7d
DV		 1131 spin_lock_irq(&dev_priv->irq_lock);
1132 valleyview_enable_display_irqs(dev_priv);
1133 spin_unlock_irq(&dev_priv->irq_lock);
1134
1135 /*
1136 * During driver initialization/resume we can avoid restoring the
1137 * part of the HW/SW state that will be inited anyway explicitly.
1138 */
1139 if (dev_priv->power_domains.initializing)
1140 return;
1141
b963291c 1142 intel_hpd_init(dev_priv);
9c065a7d 1143
9504a892
L		 1144 /* Re-enable the ADPA, if we have one */
1145 for_each_intel_encoder(&dev_priv->drm, encoder) {
1146 if (encoder->type == INTEL_OUTPUT_ANALOG)
1147 intel_crt_reset(&encoder->base);
1148 }
1149
29b74b7f 1150 i915_redisable_vga_power_on(dev_priv);
8090ba8c
ID		 1151
1152 intel_pps_unlock_regs_wa(dev_priv);
9c065a7d
DV		 1153}
1154
2be7d540
VS		 1155static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
1156{
1157 spin_lock_irq(&dev_priv->irq_lock);
1158 valleyview_disable_display_irqs(dev_priv);
1159 spin_unlock_irq(&dev_priv->irq_lock);
1160
2230fde8 1161 /* make sure we're done processing display irqs */
91c8a326 1162 synchronize_irq(dev_priv->drm.irq);
2230fde8 1163
78597996 1164 intel_power_sequencer_reset(dev_priv);
19625e85 1165
b64b5409
L		 1166 /* Prevent us from re-enabling polling on accident in late suspend */
1167 if (!dev_priv->drm.dev->power.is_suspended)
1168 intel_hpd_poll_init(dev_priv);
2be7d540
VS		 1169}
1170
1171static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
1172 struct i915_power_well *power_well)
1173{
01c3faa7 1174 WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
2be7d540
VS		 1175
1176 vlv_set_power_well(dev_priv, power_well, true);
1177
1178 vlv_display_power_well_init(dev_priv);
1179}
1180
9c065a7d
DV		 1181static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
1182 struct i915_power_well *power_well)
1183{
01c3faa7 1184 WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
9c065a7d 1185
2be7d540 1186 vlv_display_power_well_deinit(dev_priv);
9c065a7d
DV		 1187
1188 vlv_set_power_well(dev_priv, power_well, false);
9c065a7d
DV		 1189}
1190
1191static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1192 struct i915_power_well *power_well)
1193{
01c3faa7 1194 WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
9c065a7d 1195
5a8fbb7d 1196 /* since ref/cri clock was enabled */
9c065a7d
DV		 1197 udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1198
1199 vlv_set_power_well(dev_priv, power_well, true);
1200
1201 /*
1202 * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
1203 * 6. De-assert cmn_reset/side_reset. Same as VLV X0.
1204 * a. GUnit 0x2110 bit[0] set to 1 (def 0)
1205 * b. The other bits such as sfr settings / modesel may all
1206 * be set to 0.
1207 *
1208 * This should only be done on init and resume from S3 with
1209 * both PLLs disabled, or we risk losing DPIO and PLL
1210 * synchronization.
1211 */
1212 I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
1213}
1214
1215static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1216 struct i915_power_well *power_well)
1217{
1218 enum pipe pipe;
1219
01c3faa7 1220 WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
9c065a7d
DV		 1221
1222 for_each_pipe(dev_priv, pipe)
1223 assert_pll_disabled(dev_priv, pipe);
1224
1225 /* Assert common reset */
1226 I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
1227
1228 vlv_set_power_well(dev_priv, power_well, false);
1229}
1230
d8fc70b7 1231#define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
30142273
VS		 1232
1233static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv,
1234 int power_well_id)
1235{
1236 struct i915_power_domains *power_domains = &dev_priv->power_domains;
30142273
VS		 1237 int i;
1238
fc17f227
ID		 1239 for (i = 0; i < power_domains->power_well_count; i++) {
1240 struct i915_power_well *power_well;
1241
1242 power_well = &power_domains->power_wells[i];
01c3faa7 1243 if (power_well->id == power_well_id)
30142273
VS		 1244 return power_well;
1245 }
1246
1247 return NULL;
1248}
1249
1250#define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1251
1252static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1253{
1254 struct i915_power_well *cmn_bc =
1255 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
1256 struct i915_power_well *cmn_d =
1257 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
1258 u32 phy_control = dev_priv->chv_phy_control;
1259 u32 phy_status = 0;
3be60de9 1260 u32 phy_status_mask = 0xffffffff;
30142273 1261
3be60de9
VS		 1262 /*
1263 * The BIOS can leave the PHY is some weird state
1264 * where it doesn't fully power down some parts.
1265 * Disable the asserts until the PHY has been fully
1266 * reset (ie. the power well has been disabled at
1267 * least once).
1268 */
1269 if (!dev_priv->chv_phy_assert[DPIO_PHY0])
1270 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1271 PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1272 PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1273 PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1274 PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1275 PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1276
1277 if (!dev_priv->chv_phy_assert[DPIO_PHY1])
1278 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1279 PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1280 PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1281
30142273
VS		 1282 if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
1283 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1284
1285 /* this assumes override is only used to enable lanes */
1286 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1287 phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1288
1289 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1290 phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1291
1292 /* CL1 is on whenever anything is on in either channel */
1293 if (BITS_SET(phy_control,
1294 PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1295 PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1296 phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1297
1298 /*
1299 * The DPLLB check accounts for the pipe B + port A usage
1300 * with CL2 powered up but all the lanes in the second channel
1301 * powered down.
1302 */
1303 if (BITS_SET(phy_control,
1304 PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1305 (I915_READ(DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1306 phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1307
1308 if (BITS_SET(phy_control,
1309 PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1310 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1311 if (BITS_SET(phy_control,
1312 PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1313 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1314
1315 if (BITS_SET(phy_control,
1316 PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1317 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1318 if (BITS_SET(phy_control,
1319 PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1320 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1321 }
1322
1323 if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
1324 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1325
1326 /* this assumes override is only used to enable lanes */
1327 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1328 phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1329
1330 if (BITS_SET(phy_control,
1331 PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1332 phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1333
1334 if (BITS_SET(phy_control,
1335 PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1336 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1337 if (BITS_SET(phy_control,
1338 PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1339 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1340 }
1341
3be60de9
VS		 1342 phy_status &= phy_status_mask;
1343
30142273
VS		 1344 /*
1345 * The PHY may be busy with some initial calibration and whatnot,
1346 * so the power state can take a while to actually change.
1347 */
919fcd51
CW		 1348 if (intel_wait_for_register(dev_priv,
1349 DISPLAY_PHY_STATUS,
1350 phy_status_mask,
1351 phy_status,
1352 10))
1353 DRM_ERROR("Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1354 I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask,
1355 phy_status, dev_priv->chv_phy_control);
30142273
VS		 1356}
1357
1358#undef BITS_SET
1359
9c065a7d
DV		 1360static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1361 struct i915_power_well *power_well)
1362{
1363 enum dpio_phy phy;
e0fce78f
VS		 1364 enum pipe pipe;
1365 uint32_t tmp;
9c065a7d 1366
01c3faa7
ACO		 1367 WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1368 power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
9c065a7d 1369
01c3faa7 1370 if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
e0fce78f 1371 pipe = PIPE_A;
9c065a7d 1372 phy = DPIO_PHY0;
e0fce78f
VS		 1373 } else {
1374 pipe = PIPE_C;
9c065a7d 1375 phy = DPIO_PHY1;
e0fce78f 1376 }
5a8fbb7d
VS		 1377
1378 /* since ref/cri clock was enabled */
9c065a7d
DV		 1379 udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1380 vlv_set_power_well(dev_priv, power_well, true);
1381
1382 /* Poll for phypwrgood signal */
ffebb83b
CW		 1383 if (intel_wait_for_register(dev_priv,
1384 DISPLAY_PHY_STATUS,
1385 PHY_POWERGOOD(phy),
1386 PHY_POWERGOOD(phy),
1387 1))
9c065a7d
DV		 1388 DRM_ERROR("Display PHY %d is not power up\n", phy);
1389
e0fce78f
VS		 1390 mutex_lock(&dev_priv->sb_lock);
1391
1392 /* Enable dynamic power down */
1393 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
ee279218
VS		 1394 tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1395 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
e0fce78f
VS		 1396 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
1397
01c3faa7 1398 if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
e0fce78f
VS		 1399 tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
1400 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1401 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
3e288786
VS		 1402 } else {
1403 /*
1404 * Force the non-existing CL2 off. BXT does this
1405 * too, so maybe it saves some power even though
1406 * CL2 doesn't exist?
1407 */
1408 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
1409 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1410 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
e0fce78f
VS		 1411 }
1412
1413 mutex_unlock(&dev_priv->sb_lock);
1414
70722468
VS		 1415 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1416 I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
e0fce78f
VS		 1417
1418 DRM_DEBUG_KMS("Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1419 phy, dev_priv->chv_phy_control);
30142273
VS		 1420
1421 assert_chv_phy_status(dev_priv);
9c065a7d
DV		 1422}
1423
1424static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1425 struct i915_power_well *power_well)
1426{
1427 enum dpio_phy phy;
1428
01c3faa7
ACO		 1429 WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1430 power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
9c065a7d 1431
01c3faa7 1432 if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
9c065a7d
DV		 1433 phy = DPIO_PHY0;
1434 assert_pll_disabled(dev_priv, PIPE_A);
1435 assert_pll_disabled(dev_priv, PIPE_B);
1436 } else {
1437 phy = DPIO_PHY1;
1438 assert_pll_disabled(dev_priv, PIPE_C);
1439 }
1440
70722468
VS		 1441 dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1442 I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
9c065a7d
DV		 1443
1444 vlv_set_power_well(dev_priv, power_well, false);
e0fce78f
VS		 1445
1446 DRM_DEBUG_KMS("Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1447 phy, dev_priv->chv_phy_control);
30142273 1448
3be60de9
VS		 1449 /* PHY is fully reset now, so we can enable the PHY state asserts */
1450 dev_priv->chv_phy_assert[phy] = true;
1451
30142273 1452 assert_chv_phy_status(dev_priv);
e0fce78f
VS		 1453}
1454
6669e39f
VS		 1455static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1456 enum dpio_channel ch, bool override, unsigned int mask)
1457{
1458 enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
1459 u32 reg, val, expected, actual;
1460
3be60de9
VS		 1461 /*
1462 * The BIOS can leave the PHY is some weird state
1463 * where it doesn't fully power down some parts.
1464 * Disable the asserts until the PHY has been fully
1465 * reset (ie. the power well has been disabled at
1466 * least once).
1467 */
1468 if (!dev_priv->chv_phy_assert[phy])
1469 return;
1470
6669e39f
VS		 1471 if (ch == DPIO_CH0)
1472 reg = _CHV_CMN_DW0_CH0;
1473 else
1474 reg = _CHV_CMN_DW6_CH1;
1475
1476 mutex_lock(&dev_priv->sb_lock);
1477 val = vlv_dpio_read(dev_priv, pipe, reg);
1478 mutex_unlock(&dev_priv->sb_lock);
1479
1480 /*
1481 * This assumes !override is only used when the port is disabled.
1482 * All lanes should power down even without the override when
1483 * the port is disabled.
1484 */
1485 if (!override || mask == 0xf) {
1486 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1487 /*
1488 * If CH1 common lane is not active anymore
1489 * (eg. for pipe B DPLL) the entire channel will
1490 * shut down, which causes the common lane registers
1491 * to read as 0. That means we can't actually check
1492 * the lane power down status bits, but as the entire
1493 * register reads as 0 it's a good indication that the
1494 * channel is indeed entirely powered down.
1495 */
1496 if (ch == DPIO_CH1 && val == 0)
1497 expected = 0;
1498 } else if (mask != 0x0) {
1499 expected = DPIO_ANYDL_POWERDOWN;
1500 } else {
1501 expected = 0;
1502 }
1503
1504 if (ch == DPIO_CH0)
1505 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1506 else
1507 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1508 actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1509
1510 WARN(actual != expected,
1511 "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1512 !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN),
1513 !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN),
1514 reg, val);
1515}
1516
b0b33846
VS		 1517bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1518 enum dpio_channel ch, bool override)
1519{
1520 struct i915_power_domains *power_domains = &dev_priv->power_domains;
1521 bool was_override;
1522
1523 mutex_lock(&power_domains->lock);
1524
1525 was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1526
1527 if (override == was_override)
1528 goto out;
1529
1530 if (override)
1531 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1532 else
1533 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1534
1535 I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1536
1537 DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1538 phy, ch, dev_priv->chv_phy_control);
1539
30142273
VS		 1540 assert_chv_phy_status(dev_priv);
1541
b0b33846
VS		 1542out:
1543 mutex_unlock(&power_domains->lock);
1544
1545 return was_override;
1546}
1547
e0fce78f
VS		 1548void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1549 bool override, unsigned int mask)
1550{
1551 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1552 struct i915_power_domains *power_domains = &dev_priv->power_domains;
1553 enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(&encoder->base));
1554 enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
1555
1556 mutex_lock(&power_domains->lock);
1557
1558 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1559 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1560
1561 if (override)
1562 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1563 else
1564 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1565
1566 I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1567
1568 DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1569 phy, ch, mask, dev_priv->chv_phy_control);
1570
30142273
VS		 1571 assert_chv_phy_status(dev_priv);
1572
6669e39f
VS		 1573 assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1574
e0fce78f 1575 mutex_unlock(&power_domains->lock);
9c065a7d
DV		 1576}
1577
1578static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1579 struct i915_power_well *power_well)
1580{
01c3faa7 1581 enum pipe pipe = power_well->id;
9c065a7d
DV		 1582 bool enabled;
1583 u32 state, ctrl;
1584
1585 mutex_lock(&dev_priv->rps.hw_lock);
1586
1587 state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
1588 /*
1589 * We only ever set the power-on and power-gate states, anything
1590 * else is unexpected.
1591 */
1592 WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
1593 enabled = state == DP_SSS_PWR_ON(pipe);
1594
1595 /*
1596 * A transient state at this point would mean some unexpected party
1597 * is poking at the power controls too.
1598 */
1599 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
1600 WARN_ON(ctrl << 16 != state);
1601
1602 mutex_unlock(&dev_priv->rps.hw_lock);
1603
1604 return enabled;
1605}
1606
1607static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1608 struct i915_power_well *power_well,
1609 bool enable)
1610{
01c3faa7 1611 enum pipe pipe = power_well->id;
9c065a7d
DV		 1612 u32 state;
1613 u32 ctrl;
1614
1615 state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1616
1617 mutex_lock(&dev_priv->rps.hw_lock);
1618
1619#define COND \
1620 ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
1621
1622 if (COND)
1623 goto out;
1624
1625 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
1626 ctrl &= ~DP_SSC_MASK(pipe);
1627 ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1628 vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
1629
1630 if (wait_for(COND, 100))
7e35ab88 1631 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
9c065a7d
DV		 1632 state,
1633 vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
1634
1635#undef COND
1636
1637out:
1638 mutex_unlock(&dev_priv->rps.hw_lock);
1639}
1640
9c065a7d
DV		 1641static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1642 struct i915_power_well *power_well)
1643{
01c3faa7 1644 WARN_ON_ONCE(power_well->id != PIPE_A);
9c065a7d
DV		 1645
1646 chv_set_pipe_power_well(dev_priv, power_well, true);
afd6275d 1647
2be7d540 1648 vlv_display_power_well_init(dev_priv);
9c065a7d
DV		 1649}
1650
1651static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1652 struct i915_power_well *power_well)
1653{
01c3faa7 1654 WARN_ON_ONCE(power_well->id != PIPE_A);
8fcd5cd8 1655
2be7d540 1656 vlv_display_power_well_deinit(dev_priv);
afd6275d 1657
9c065a7d
DV		 1658 chv_set_pipe_power_well(dev_priv, power_well, false);
1659}
1660
09731280
ID		 1661static void
1662__intel_display_power_get_domain(struct drm_i915_private *dev_priv,
1663 enum intel_display_power_domain domain)
1664{
1665 struct i915_power_domains *power_domains = &dev_priv->power_domains;
1666 struct i915_power_well *power_well;
09731280 1667
75ccb2ec 1668 for_each_power_domain_well(dev_priv, power_well, BIT_ULL(domain))
b409ca95 1669 intel_power_well_get(dev_priv, power_well);
09731280
ID		 1670
1671 power_domains->domain_use_count[domain]++;
1672}
1673
e4e7684f
DV		 1674/**
1675 * intel_display_power_get - grab a power domain reference
1676 * @dev_priv: i915 device instance
1677 * @domain: power domain to reference
1678 *
1679 * This function grabs a power domain reference for @domain and ensures that the
1680 * power domain and all its parents are powered up. Therefore users should only
1681 * grab a reference to the innermost power domain they need.
1682 *
1683 * Any power domain reference obtained by this function must have a symmetric
1684 * call to intel_display_power_put() to release the reference again.
1685 */
9c065a7d
DV		 1686void intel_display_power_get(struct drm_i915_private *dev_priv,
1687 enum intel_display_power_domain domain)
1688{
09731280 1689 struct i915_power_domains *power_domains = &dev_priv->power_domains;
9c065a7d
DV		 1690
1691 intel_runtime_pm_get(dev_priv);
1692
09731280
ID		 1693 mutex_lock(&power_domains->lock);
1694
1695 __intel_display_power_get_domain(dev_priv, domain);
1696
1697 mutex_unlock(&power_domains->lock);
1698}
1699
1700/**
1701 * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
1702 * @dev_priv: i915 device instance
1703 * @domain: power domain to reference
1704 *
1705 * This function grabs a power domain reference for @domain and ensures that the
1706 * power domain and all its parents are powered up. Therefore users should only
1707 * grab a reference to the innermost power domain they need.
1708 *
1709 * Any power domain reference obtained by this function must have a symmetric
1710 * call to intel_display_power_put() to release the reference again.
1711 */
1712bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
1713 enum intel_display_power_domain domain)
1714{
1715 struct i915_power_domains *power_domains = &dev_priv->power_domains;
1716 bool is_enabled;
1717
1718 if (!intel_runtime_pm_get_if_in_use(dev_priv))
1719 return false;
9c065a7d
DV		 1720
1721 mutex_lock(&power_domains->lock);
1722
09731280
ID		 1723 if (__intel_display_power_is_enabled(dev_priv, domain)) {
1724 __intel_display_power_get_domain(dev_priv, domain);
1725 is_enabled = true;
1726 } else {
1727 is_enabled = false;
9c065a7d
DV		 1728 }
1729
9c065a7d 1730 mutex_unlock(&power_domains->lock);
09731280
ID		 1731
1732 if (!is_enabled)
1733 intel_runtime_pm_put(dev_priv);
1734
1735 return is_enabled;
9c065a7d
DV		 1736}
1737
e4e7684f
DV		 1738/**
1739 * intel_display_power_put - release a power domain reference
1740 * @dev_priv: i915 device instance
1741 * @domain: power domain to reference
1742 *
1743 * This function drops the power domain reference obtained by
1744 * intel_display_power_get() and might power down the corresponding hardware
1745 * block right away if this is the last reference.
1746 */
9c065a7d
DV		 1747void intel_display_power_put(struct drm_i915_private *dev_priv,
1748 enum intel_display_power_domain domain)
1749{
1750 struct i915_power_domains *power_domains;
1751 struct i915_power_well *power_well;
9c065a7d
DV		 1752
1753 power_domains = &dev_priv->power_domains;
1754
1755 mutex_lock(&power_domains->lock);
1756
11c86db8
DS		 1757 WARN(!power_domains->domain_use_count[domain],
1758 "Use count on domain %s is already zero\n",
1759 intel_display_power_domain_str(domain));
9c065a7d
DV		 1760 power_domains->domain_use_count[domain]--;
1761
75ccb2ec 1762 for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain))
b409ca95 1763 intel_power_well_put(dev_priv, power_well);
9c065a7d
DV		 1764
1765 mutex_unlock(&power_domains->lock);
1766
1767 intel_runtime_pm_put(dev_priv);
1768}
1769
9d0996b5 1770#define HSW_DISPLAY_POWER_DOMAINS ( \
d8fc70b7
ACO		 1771 BIT_ULL(POWER_DOMAIN_PIPE_B) | \
1772 BIT_ULL(POWER_DOMAIN_PIPE_C) | \
1773 BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) | \
1774 BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \
1775 BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) | \
1776 BIT_ULL(POWER_DOMAIN_TRANSCODER_A) | \
1777 BIT_ULL(POWER_DOMAIN_TRANSCODER_B) | \
1778 BIT_ULL(POWER_DOMAIN_TRANSCODER_C) | \
1779 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
1780 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
1781 BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) | \
1782 BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */ \
1783 BIT_ULL(POWER_DOMAIN_VGA) | \
1784 BIT_ULL(POWER_DOMAIN_AUDIO) | \
1785 BIT_ULL(POWER_DOMAIN_INIT))
9c065a7d 1786
9d0996b5 1787#define BDW_DISPLAY_POWER_DOMAINS ( \
d8fc70b7
ACO		 1788 BIT_ULL(POWER_DOMAIN_PIPE_B) | \
1789 BIT_ULL(POWER_DOMAIN_PIPE_C) | \
1790 BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \
1791 BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) | \
1792 BIT_ULL(POWER_DOMAIN_TRANSCODER_A) | \
1793 BIT_ULL(POWER_DOMAIN_TRANSCODER_B) | \
1794 BIT_ULL(POWER_DOMAIN_TRANSCODER_C) | \
1795 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
1796 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
1797 BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) | \
1798 BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */ \
1799 BIT_ULL(POWER_DOMAIN_VGA) | \
1800 BIT_ULL(POWER_DOMAIN_AUDIO) | \
1801 BIT_ULL(POWER_DOMAIN_INIT))
9c065a7d 1802
465ac0c6 1803#define VLV_DISPLAY_POWER_DOMAINS ( \
d8fc70b7
ACO		 1804 BIT_ULL(POWER_DOMAIN_PIPE_A) | \
1805 BIT_ULL(POWER_DOMAIN_PIPE_B) | \
1806 BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) | \
1807 BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \
1808 BIT_ULL(POWER_DOMAIN_TRANSCODER_A) | \
1809 BIT_ULL(POWER_DOMAIN_TRANSCODER_B) | \
1810 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
1811 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
1812 BIT_ULL(POWER_DOMAIN_PORT_DSI) | \
1813 BIT_ULL(POWER_DOMAIN_PORT_CRT) | \
1814 BIT_ULL(POWER_DOMAIN_VGA) | \
1815 BIT_ULL(POWER_DOMAIN_AUDIO) | \
1816 BIT_ULL(POWER_DOMAIN_AUX_B) | \
1817 BIT_ULL(POWER_DOMAIN_AUX_C) | \
1818 BIT_ULL(POWER_DOMAIN_GMBUS) | \
1819 BIT_ULL(POWER_DOMAIN_INIT))
9c065a7d
DV		 1820
1821#define VLV_DPIO_CMN_BC_POWER_DOMAINS ( \
d8fc70b7
ACO		 1822 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
1823 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
1824 BIT_ULL(POWER_DOMAIN_PORT_CRT) | \
1825 BIT_ULL(POWER_DOMAIN_AUX_B) | \
1826 BIT_ULL(POWER_DOMAIN_AUX_C) | \
1827 BIT_ULL(POWER_DOMAIN_INIT))
9c065a7d
DV		 1828
1829#define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS ( \
d8fc70b7
ACO		 1830 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
1831 BIT_ULL(POWER_DOMAIN_AUX_B) | \
1832 BIT_ULL(POWER_DOMAIN_INIT))
9c065a7d
DV		 1833
1834#define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS ( \
d8fc70b7
ACO		 1835 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
1836 BIT_ULL(POWER_DOMAIN_AUX_B) | \
1837 BIT_ULL(POWER_DOMAIN_INIT))
9c065a7d
DV		 1838
1839#define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS ( \
d8fc70b7
ACO		 1840 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
1841 BIT_ULL(POWER_DOMAIN_AUX_C) | \
1842 BIT_ULL(POWER_DOMAIN_INIT))
9c065a7d
DV		 1843
1844#define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS ( \
d8fc70b7
ACO		 1845 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
1846 BIT_ULL(POWER_DOMAIN_AUX_C) | \
1847 BIT_ULL(POWER_DOMAIN_INIT))
9c065a7d 1848
465ac0c6 1849#define CHV_DISPLAY_POWER_DOMAINS ( \
d8fc70b7
ACO		 1850 BIT_ULL(POWER_DOMAIN_PIPE_A) | \
1851 BIT_ULL(POWER_DOMAIN_PIPE_B) | \
1852 BIT_ULL(POWER_DOMAIN_PIPE_C) | \
1853 BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) | \
1854 BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \
1855 BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) | \
1856 BIT_ULL(POWER_DOMAIN_TRANSCODER_A) | \
1857 BIT_ULL(POWER_DOMAIN_TRANSCODER_B) | \
1858 BIT_ULL(POWER_DOMAIN_TRANSCODER_C) | \
1859 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
1860 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
1861 BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) | \
1862 BIT_ULL(POWER_DOMAIN_PORT_DSI) | \
1863 BIT_ULL(POWER_DOMAIN_VGA) | \
1864 BIT_ULL(POWER_DOMAIN_AUDIO) | \
1865 BIT_ULL(POWER_DOMAIN_AUX_B) | \
1866 BIT_ULL(POWER_DOMAIN_AUX_C) | \
1867 BIT_ULL(POWER_DOMAIN_AUX_D) | \
1868 BIT_ULL(POWER_DOMAIN_GMBUS) | \
1869 BIT_ULL(POWER_DOMAIN_INIT))
465ac0c6 1870
9c065a7d 1871#define CHV_DPIO_CMN_BC_POWER_DOMAINS ( \
d8fc70b7
ACO		 1872 BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) | \
1873 BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) | \
1874 BIT_ULL(POWER_DOMAIN_AUX_B) | \
1875 BIT_ULL(POWER_DOMAIN_AUX_C) | \
1876 BIT_ULL(POWER_DOMAIN_INIT))
9c065a7d
DV		 1877
1878#define CHV_DPIO_CMN_D_POWER_DOMAINS ( \
d8fc70b7
ACO		 1879 BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) | \
1880 BIT_ULL(POWER_DOMAIN_AUX_D) | \
1881 BIT_ULL(POWER_DOMAIN_INIT))
9c065a7d 1882
9c065a7d 1883static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
3c1b38e6 1884 .sync_hw = i9xx_power_well_sync_hw_noop,
9c065a7d
DV		 1885 .enable = i9xx_always_on_power_well_noop,
1886 .disable = i9xx_always_on_power_well_noop,
1887 .is_enabled = i9xx_always_on_power_well_enabled,
1888};
1889
1890static const struct i915_power_well_ops chv_pipe_power_well_ops = {
3c1b38e6 1891 .sync_hw = i9xx_power_well_sync_hw_noop,
9c065a7d
DV		 1892 .enable = chv_pipe_power_well_enable,
1893 .disable = chv_pipe_power_well_disable,
1894 .is_enabled = chv_pipe_power_well_enabled,
1895};
1896
1897static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
3c1b38e6 1898 .sync_hw = i9xx_power_well_sync_hw_noop,
9c065a7d
DV		 1899 .enable = chv_dpio_cmn_power_well_enable,
1900 .disable = chv_dpio_cmn_power_well_disable,
1901 .is_enabled = vlv_power_well_enabled,
1902};
1903
1904static struct i915_power_well i9xx_always_on_power_well[] = {
1905 {
1906 .name = "always-on",
1907 .always_on = 1,
1908 .domains = POWER_DOMAIN_MASK,
1909 .ops = &i9xx_always_on_power_well_ops,
1910 },
1911};
1912
1913static const struct i915_power_well_ops hsw_power_well_ops = {
1914 .sync_hw = hsw_power_well_sync_hw,
1915 .enable = hsw_power_well_enable,
1916 .disable = hsw_power_well_disable,
1917 .is_enabled = hsw_power_well_enabled,
1918};
1919
94dd5138
S		 1920static const struct i915_power_well_ops skl_power_well_ops = {
1921 .sync_hw = skl_power_well_sync_hw,
1922 .enable = skl_power_well_enable,
1923 .disable = skl_power_well_disable,
1924 .is_enabled = skl_power_well_enabled,
1925};
1926
9f836f90 1927static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
3c1b38e6 1928 .sync_hw = i9xx_power_well_sync_hw_noop,
9f836f90
PJ		 1929 .enable = gen9_dc_off_power_well_enable,
1930 .disable = gen9_dc_off_power_well_disable,
1931 .is_enabled = gen9_dc_off_power_well_enabled,
1932};
1933
9c8d0b8e 1934static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
3c1b38e6 1935 .sync_hw = i9xx_power_well_sync_hw_noop,
9c8d0b8e
ID		 1936 .enable = bxt_dpio_cmn_power_well_enable,
1937 .disable = bxt_dpio_cmn_power_well_disable,
1938 .is_enabled = bxt_dpio_cmn_power_well_enabled,
1939};
1940
9c065a7d
DV		 1941static struct i915_power_well hsw_power_wells[] = {
1942 {
1943 .name = "always-on",
1944 .always_on = 1,
998bd66a 1945 .domains = POWER_DOMAIN_MASK,
9c065a7d
DV		 1946 .ops = &i9xx_always_on_power_well_ops,
1947 },
1948 {
1949 .name = "display",
1950 .domains = HSW_DISPLAY_POWER_DOMAINS,
1951 .ops = &hsw_power_well_ops,
1952 },
1953};
1954
1955static struct i915_power_well bdw_power_wells[] = {
1956 {
1957 .name = "always-on",
1958 .always_on = 1,
998bd66a 1959 .domains = POWER_DOMAIN_MASK,
9c065a7d
DV		 1960 .ops = &i9xx_always_on_power_well_ops,
1961 },
1962 {
1963 .name = "display",
1964 .domains = BDW_DISPLAY_POWER_DOMAINS,
1965 .ops = &hsw_power_well_ops,
1966 },
1967};
1968
1969static const struct i915_power_well_ops vlv_display_power_well_ops = {
3c1b38e6 1970 .sync_hw = i9xx_power_well_sync_hw_noop,
9c065a7d
DV		 1971 .enable = vlv_display_power_well_enable,
1972 .disable = vlv_display_power_well_disable,
1973 .is_enabled = vlv_power_well_enabled,
1974};
1975
1976static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
3c1b38e6 1977 .sync_hw = i9xx_power_well_sync_hw_noop,
9c065a7d
DV		 1978 .enable = vlv_dpio_cmn_power_well_enable,
1979 .disable = vlv_dpio_cmn_power_well_disable,
1980 .is_enabled = vlv_power_well_enabled,
1981};
1982
1983static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
3c1b38e6 1984 .sync_hw = i9xx_power_well_sync_hw_noop,
9c065a7d
DV		 1985 .enable = vlv_power_well_enable,
1986 .disable = vlv_power_well_disable,
1987 .is_enabled = vlv_power_well_enabled,
1988};
1989
1990static struct i915_power_well vlv_power_wells[] = {
1991 {
1992 .name = "always-on",
1993 .always_on = 1,
998bd66a 1994 .domains = POWER_DOMAIN_MASK,
9c065a7d 1995 .ops = &i9xx_always_on_power_well_ops,
01c3faa7 1996 .id = PUNIT_POWER_WELL_ALWAYS_ON,
9c065a7d
DV		 1997 },
1998 {
1999 .name = "display",
2000 .domains = VLV_DISPLAY_POWER_DOMAINS,
01c3faa7 2001 .id = PUNIT_POWER_WELL_DISP2D,
9c065a7d
DV		 2002 .ops = &vlv_display_power_well_ops,
2003 },
2004 {
2005 .name = "dpio-tx-b-01",
2006 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2007 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2008 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2009 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2010 .ops = &vlv_dpio_power_well_ops,
01c3faa7 2011 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
9c065a7d
DV		 2012 },
2013 {
2014 .name = "dpio-tx-b-23",
2015 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2016 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2017 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2018 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2019 .ops = &vlv_dpio_power_well_ops,
01c3faa7 2020 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
9c065a7d
DV		 2021 },
2022 {
2023 .name = "dpio-tx-c-01",
2024 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2025 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2026 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2027 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2028 .ops = &vlv_dpio_power_well_ops,
01c3faa7 2029 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
9c065a7d
DV		 2030 },
2031 {
2032 .name = "dpio-tx-c-23",
2033 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2034 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2035 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2036 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2037 .ops = &vlv_dpio_power_well_ops,
01c3faa7 2038 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
9c065a7d
DV		 2039 },
2040 {
2041 .name = "dpio-common",
2042 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
01c3faa7 2043 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
9c065a7d
DV		 2044 .ops = &vlv_dpio_cmn_power_well_ops,
2045 },
2046};
2047
2048static struct i915_power_well chv_power_wells[] = {
2049 {
2050 .name = "always-on",
2051 .always_on = 1,
998bd66a 2052 .domains = POWER_DOMAIN_MASK,
9c065a7d
DV		 2053 .ops = &i9xx_always_on_power_well_ops,
2054 },
9c065a7d
DV		 2055 {
2056 .name = "display",
baa4e575 2057 /*
fde61e4b
VS		 2058 * Pipe A power well is the new disp2d well. Pipe B and C
2059 * power wells don't actually exist. Pipe A power well is
2060 * required for any pipe to work.
baa4e575 2061 */
465ac0c6 2062 .domains = CHV_DISPLAY_POWER_DOMAINS,
01c3faa7 2063 .id = PIPE_A,
9c065a7d
DV		 2064 .ops = &chv_pipe_power_well_ops,
2065 },
9c065a7d
DV		 2066 {
2067 .name = "dpio-common-bc",
71849b67 2068 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
01c3faa7 2069 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
9c065a7d
DV		 2070 .ops = &chv_dpio_cmn_power_well_ops,
2071 },
2072 {
2073 .name = "dpio-common-d",
71849b67 2074 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
01c3faa7 2075 .id = PUNIT_POWER_WELL_DPIO_CMN_D,
9c065a7d
DV		 2076 .ops = &chv_dpio_cmn_power_well_ops,
2077 },
9c065a7d
DV		 2078};
2079
5aefb239
SS		 2080bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
2081 int power_well_id)
2082{
2083 struct i915_power_well *power_well;
2084 bool ret;
2085
2086 power_well = lookup_power_well(dev_priv, power_well_id);
2087 ret = power_well->ops->is_enabled(dev_priv, power_well);
2088
2089 return ret;
2090}
2091
94dd5138
S		 2092static struct i915_power_well skl_power_wells[] = {
2093 {
2094 .name = "always-on",
2095 .always_on = 1,
998bd66a 2096 .domains = POWER_DOMAIN_MASK,
94dd5138 2097 .ops = &i9xx_always_on_power_well_ops,
01c3faa7 2098 .id = SKL_DISP_PW_ALWAYS_ON,
94dd5138
S		 2099 },
2100 {
2101 .name = "power well 1",
4a76f295
ID		 2102 /* Handled by the DMC firmware */
2103 .domains = 0,
94dd5138 2104 .ops = &skl_power_well_ops,
01c3faa7 2105 .id = SKL_DISP_PW_1,
94dd5138
S		 2106 },
2107 {
2108 .name = "MISC IO power well",
4a76f295
ID		 2109 /* Handled by the DMC firmware */
2110 .domains = 0,
94dd5138 2111 .ops = &skl_power_well_ops,
01c3faa7 2112 .id = SKL_DISP_PW_MISC_IO,
94dd5138 2113 },
9f836f90
PJ		 2114 {
2115 .name = "DC off",
2116 .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
2117 .ops = &gen9_dc_off_power_well_ops,
01c3faa7 2118 .id = SKL_DISP_PW_DC_OFF,
9f836f90 2119 },
94dd5138
S		 2120 {
2121 .name = "power well 2",
2122 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2123 .ops = &skl_power_well_ops,
01c3faa7 2124 .id = SKL_DISP_PW_2,
94dd5138
S		 2125 },
2126 {
62b69566
ACO		 2127 .name = "DDI A/E IO power well",
2128 .domains = SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS,
94dd5138 2129 .ops = &skl_power_well_ops,
01c3faa7 2130 .id = SKL_DISP_PW_DDI_A_E,
94dd5138
S		 2131 },
2132 {
62b69566
ACO		 2133 .name = "DDI B IO power well",
2134 .domains = SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS,
94dd5138 2135 .ops = &skl_power_well_ops,
01c3faa7 2136 .id = SKL_DISP_PW_DDI_B,
94dd5138
S		 2137 },
2138 {
62b69566
ACO		 2139 .name = "DDI C IO power well",
2140 .domains = SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS,
94dd5138 2141 .ops = &skl_power_well_ops,
01c3faa7 2142 .id = SKL_DISP_PW_DDI_C,
94dd5138
S		 2143 },
2144 {
62b69566
ACO		 2145 .name = "DDI D IO power well",
2146 .domains = SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS,
94dd5138 2147 .ops = &skl_power_well_ops,
01c3faa7 2148 .id = SKL_DISP_PW_DDI_D,
94dd5138
S		 2149 },
2150};
2151
0b4a2a36
S		 2152static struct i915_power_well bxt_power_wells[] = {
2153 {
2154 .name = "always-on",
2155 .always_on = 1,
998bd66a 2156 .domains = POWER_DOMAIN_MASK,
0b4a2a36
S		 2157 .ops = &i9xx_always_on_power_well_ops,
2158 },
2159 {
2160 .name = "power well 1",
d7d7c9ee 2161 .domains = 0,
0b4a2a36 2162 .ops = &skl_power_well_ops,
01c3faa7 2163 .id = SKL_DISP_PW_1,
0b4a2a36 2164 },
9f836f90
PJ		 2165 {
2166 .name = "DC off",
2167 .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
2168 .ops = &gen9_dc_off_power_well_ops,
01c3faa7 2169 .id = SKL_DISP_PW_DC_OFF,
9f836f90 2170 },
0b4a2a36
S		 2171 {
2172 .name = "power well 2",
2173 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2174 .ops = &skl_power_well_ops,
01c3faa7 2175 .id = SKL_DISP_PW_2,
9f836f90 2176 },
9c8d0b8e
ID		 2177 {
2178 .name = "dpio-common-a",
2179 .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
2180 .ops = &bxt_dpio_cmn_power_well_ops,
01c3faa7 2181 .id = BXT_DPIO_CMN_A,
362624c9 2182 .data = DPIO_PHY1,
9c8d0b8e
ID		 2183 },
2184 {
2185 .name = "dpio-common-bc",
2186 .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
2187 .ops = &bxt_dpio_cmn_power_well_ops,
01c3faa7 2188 .id = BXT_DPIO_CMN_BC,
362624c9 2189 .data = DPIO_PHY0,
9c8d0b8e 2190 },
0b4a2a36
S		 2191};
2192
0d03926d
ACO		 2193static struct i915_power_well glk_power_wells[] = {
2194 {
2195 .name = "always-on",
2196 .always_on = 1,
2197 .domains = POWER_DOMAIN_MASK,
2198 .ops = &i9xx_always_on_power_well_ops,
2199 },
2200 {
2201 .name = "power well 1",
2202 /* Handled by the DMC firmware */
2203 .domains = 0,
2204 .ops = &skl_power_well_ops,
2205 .id = SKL_DISP_PW_1,
2206 },
2207 {
2208 .name = "DC off",
2209 .domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
2210 .ops = &gen9_dc_off_power_well_ops,
2211 .id = SKL_DISP_PW_DC_OFF,
2212 },
2213 {
2214 .name = "power well 2",
2215 .domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2216 .ops = &skl_power_well_ops,
2217 .id = SKL_DISP_PW_2,
2218 },
0a116ce8
ACO		 2219 {
2220 .name = "dpio-common-a",
2221 .domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
2222 .ops = &bxt_dpio_cmn_power_well_ops,
2223 .id = BXT_DPIO_CMN_A,
2224 .data = DPIO_PHY1,
2225 },
2226 {
2227 .name = "dpio-common-b",
2228 .domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
2229 .ops = &bxt_dpio_cmn_power_well_ops,
2230 .id = BXT_DPIO_CMN_BC,
2231 .data = DPIO_PHY0,
2232 },
2233 {
2234 .name = "dpio-common-c",
2235 .domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
2236 .ops = &bxt_dpio_cmn_power_well_ops,
2237 .id = GLK_DPIO_CMN_C,
2238 .data = DPIO_PHY2,
2239 },
0d03926d
ACO		 2240 {
2241 .name = "AUX A",
2242 .domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
2243 .ops = &skl_power_well_ops,
2244 .id = GLK_DISP_PW_AUX_A,
2245 },
2246 {
2247 .name = "AUX B",
2248 .domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
2249 .ops = &skl_power_well_ops,
2250 .id = GLK_DISP_PW_AUX_B,
2251 },
2252 {
2253 .name = "AUX C",
2254 .domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
2255 .ops = &skl_power_well_ops,
2256 .id = GLK_DISP_PW_AUX_C,
2257 },
2258 {
62b69566
ACO		 2259 .name = "DDI A IO power well",
2260 .domains = GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS,
0d03926d
ACO		 2261 .ops = &skl_power_well_ops,
2262 .id = GLK_DISP_PW_DDI_A,
2263 },
2264 {
62b69566
ACO		 2265 .name = "DDI B IO power well",
2266 .domains = GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS,
0d03926d
ACO		 2267 .ops = &skl_power_well_ops,
2268 .id = SKL_DISP_PW_DDI_B,
2269 },
2270 {
62b69566
ACO		 2271 .name = "DDI C IO power well",
2272 .domains = GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS,
0d03926d
ACO		 2273 .ops = &skl_power_well_ops,
2274 .id = SKL_DISP_PW_DDI_C,
2275 },
2276};
2277
1b0e3a04
ID		 2278static int
2279sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
2280 int disable_power_well)
2281{
2282 if (disable_power_well >= 0)
2283 return !!disable_power_well;
2284
1b0e3a04
ID		 2285 return 1;
2286}
2287
a37baf3b
ID		 2288static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
2289 int enable_dc)
2290{
2291 uint32_t mask;
2292 int requested_dc;
2293 int max_dc;
2294
b976dc53 2295 if (IS_GEN9_BC(dev_priv)) {
a37baf3b
ID		 2296 max_dc = 2;
2297 mask = 0;
cc3f90f0 2298 } else if (IS_GEN9_LP(dev_priv)) {
a37baf3b
ID		 2299 max_dc = 1;
2300 /*
2301 * DC9 has a separate HW flow from the rest of the DC states,
2302 * not depending on the DMC firmware. It's needed by system
2303 * suspend/resume, so allow it unconditionally.
2304 */
2305 mask = DC_STATE_EN_DC9;
2306 } else {
2307 max_dc = 0;
2308 mask = 0;
2309 }
2310
66e2c4c3
ID		 2311 if (!i915.disable_power_well)
2312 max_dc = 0;
2313
a37baf3b
ID		 2314 if (enable_dc >= 0 && enable_dc <= max_dc) {
2315 requested_dc = enable_dc;
2316 } else if (enable_dc == -1) {
2317 requested_dc = max_dc;
2318 } else if (enable_dc > max_dc && enable_dc <= 2) {
2319 DRM_DEBUG_KMS("Adjusting requested max DC state (%d->%d)\n",
2320 enable_dc, max_dc);
2321 requested_dc = max_dc;
2322 } else {
2323 DRM_ERROR("Unexpected value for enable_dc (%d)\n", enable_dc);
2324 requested_dc = max_dc;
2325 }
2326
2327 if (requested_dc > 1)
2328 mask |= DC_STATE_EN_UPTO_DC6;
2329 if (requested_dc > 0)
2330 mask |= DC_STATE_EN_UPTO_DC5;
2331
2332 DRM_DEBUG_KMS("Allowed DC state mask %02x\n", mask);
2333
2334 return mask;
2335}
2336
9c065a7d
DV		 2337#define set_power_wells(power_domains, __power_wells) ({ \
2338 (power_domains)->power_wells = (__power_wells); \
2339 (power_domains)->power_well_count = ARRAY_SIZE(__power_wells); \
2340})
2341
e4e7684f
DV		 2342/**
2343 * intel_power_domains_init - initializes the power domain structures
2344 * @dev_priv: i915 device instance
2345 *
2346 * Initializes the power domain structures for @dev_priv depending upon the
2347 * supported platform.
2348 */
9c065a7d
DV		 2349int intel_power_domains_init(struct drm_i915_private *dev_priv)
2350{
2351 struct i915_power_domains *power_domains = &dev_priv->power_domains;
2352
1b0e3a04
ID		 2353 i915.disable_power_well = sanitize_disable_power_well_option(dev_priv,
2354 i915.disable_power_well);
a37baf3b
ID		 2355 dev_priv->csr.allowed_dc_mask = get_allowed_dc_mask(dev_priv,
2356 i915.enable_dc);
1b0e3a04 2357
d8fc70b7 2358 BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
f0ab43e6 2359
9c065a7d
DV		 2360 mutex_init(&power_domains->lock);
2361
2362 /*
2363 * The enabling order will be from lower to higher indexed wells,
2364 * the disabling order is reversed.
2365 */
2d1fe073 2366 if (IS_HASWELL(dev_priv)) {
9c065a7d 2367 set_power_wells(power_domains, hsw_power_wells);
2d1fe073 2368 } else if (IS_BROADWELL(dev_priv)) {
9c065a7d 2369 set_power_wells(power_domains, bdw_power_wells);
b976dc53 2370 } else if (IS_GEN9_BC(dev_priv)) {
94dd5138 2371 set_power_wells(power_domains, skl_power_wells);
2d1fe073 2372 } else if (IS_BROXTON(dev_priv)) {
0b4a2a36 2373 set_power_wells(power_domains, bxt_power_wells);
0d03926d
ACO		 2374 } else if (IS_GEMINILAKE(dev_priv)) {
2375 set_power_wells(power_domains, glk_power_wells);
2d1fe073 2376 } else if (IS_CHERRYVIEW(dev_priv)) {
9c065a7d 2377 set_power_wells(power_domains, chv_power_wells);
2d1fe073 2378 } else if (IS_VALLEYVIEW(dev_priv)) {
9c065a7d
DV		 2379 set_power_wells(power_domains, vlv_power_wells);
2380 } else {
2381 set_power_wells(power_domains, i9xx_always_on_power_well);
2382 }
2383
2384 return 0;
2385}
2386
e4e7684f
DV		 2387/**
2388 * intel_power_domains_fini - finalizes the power domain structures
2389 * @dev_priv: i915 device instance
2390 *
2391 * Finalizes the power domain structures for @dev_priv depending upon the
2392 * supported platform. This function also disables runtime pm and ensures that
2393 * the device stays powered up so that the driver can be reloaded.
2394 */
f458ebbc 2395void intel_power_domains_fini(struct drm_i915_private *dev_priv)
9c065a7d 2396{
c49d13ee 2397 struct device *kdev = &dev_priv->drm.pdev->dev;
25b181b4 2398
aabee1bb
ID		 2399 /*
2400 * The i915.ko module is still not prepared to be loaded when
f458ebbc 2401 * the power well is not enabled, so just enable it in case
aabee1bb
ID		 2402 * we're going to unload/reload.
2403 * The following also reacquires the RPM reference the core passed
2404 * to the driver during loading, which is dropped in
2405 * intel_runtime_pm_enable(). We have to hand back the control of the
2406 * device to the core with this reference held.
2407 */
f458ebbc 2408 intel_display_set_init_power(dev_priv, true);
d314cd43
ID		 2409
2410 /* Remove the refcount we took to keep power well support disabled. */
2411 if (!i915.disable_power_well)
2412 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
25b181b4
ID		 2413
2414 /*
2415 * Remove the refcount we took in intel_runtime_pm_enable() in case
2416 * the platform doesn't support runtime PM.
2417 */
2418 if (!HAS_RUNTIME_PM(dev_priv))
c49d13ee 2419 pm_runtime_put(kdev);
9c065a7d
DV		 2420}
2421
30eade12 2422static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
9c065a7d
DV		 2423{
2424 struct i915_power_domains *power_domains = &dev_priv->power_domains;
2425 struct i915_power_well *power_well;
9c065a7d
DV		 2426
2427 mutex_lock(&power_domains->lock);
75ccb2ec 2428 for_each_power_well(dev_priv, power_well) {
9c065a7d
DV		 2429 power_well->ops->sync_hw(dev_priv, power_well);
2430 power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
2431 power_well);
2432 }
2433 mutex_unlock(&power_domains->lock);
2434}
2435
70c2c184
VS		 2436static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
2437{
2438 I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
2439 POSTING_READ(DBUF_CTL);
2440
2441 udelay(10);
2442
2443 if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
2444 DRM_ERROR("DBuf power enable timeout\n");
2445}
2446
2447static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
2448{
2449 I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
2450 POSTING_READ(DBUF_CTL);
2451
2452 udelay(10);
2453
2454 if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
2455 DRM_ERROR("DBuf power disable timeout!\n");
2456}
2457
73dfc227 2458static void skl_display_core_init(struct drm_i915_private *dev_priv,
443a93ac 2459 bool resume)
73dfc227
ID		 2460{
2461 struct i915_power_domains *power_domains = &dev_priv->power_domains;
443a93ac 2462 struct i915_power_well *well;
73dfc227
ID		 2463 uint32_t val;
2464
d26fa1d5
ID		 2465 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2466
73dfc227
ID		 2467 /* enable PCH reset handshake */
2468 val = I915_READ(HSW_NDE_RSTWRN_OPT);
2469 I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
2470
2471 /* enable PG1 and Misc I/O */
2472 mutex_lock(&power_domains->lock);
443a93ac
ID		 2473
2474 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2475 intel_power_well_enable(dev_priv, well);
2476
2477 well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
2478 intel_power_well_enable(dev_priv, well);
2479
73dfc227
ID		 2480 mutex_unlock(&power_domains->lock);
2481
73dfc227
ID		 2482 skl_init_cdclk(dev_priv);
2483
70c2c184
VS		 2484 gen9_dbuf_enable(dev_priv);
2485
9f7eb31a 2486 if (resume && dev_priv->csr.dmc_payload)
2abc525b 2487 intel_csr_load_program(dev_priv);
73dfc227
ID		 2488}
2489
2490static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
2491{
2492 struct i915_power_domains *power_domains = &dev_priv->power_domains;
443a93ac 2493 struct i915_power_well *well;
73dfc227 2494
d26fa1d5
ID		 2495 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2496
70c2c184
VS		 2497 gen9_dbuf_disable(dev_priv);
2498
73dfc227
ID		 2499 skl_uninit_cdclk(dev_priv);
2500
2501 /* The spec doesn't call for removing the reset handshake flag */
2502 /* disable PG1 and Misc I/O */
443a93ac 2503
73dfc227 2504 mutex_lock(&power_domains->lock);
443a93ac
ID		 2505
2506 well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
2507 intel_power_well_disable(dev_priv, well);
2508
2509 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2510 intel_power_well_disable(dev_priv, well);
2511
73dfc227
ID		 2512 mutex_unlock(&power_domains->lock);
2513}
2514
d7d7c9ee
ID		 2515void bxt_display_core_init(struct drm_i915_private *dev_priv,
2516 bool resume)
2517{
2518 struct i915_power_domains *power_domains = &dev_priv->power_domains;
2519 struct i915_power_well *well;
2520 uint32_t val;
2521
2522 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2523
2524 /*
2525 * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
2526 * or else the reset will hang because there is no PCH to respond.
2527 * Move the handshake programming to initialization sequence.
2528 * Previously was left up to BIOS.
2529 */
2530 val = I915_READ(HSW_NDE_RSTWRN_OPT);
2531 val &= ~RESET_PCH_HANDSHAKE_ENABLE;
2532 I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2533
2534 /* Enable PG1 */
2535 mutex_lock(&power_domains->lock);
2536
2537 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2538 intel_power_well_enable(dev_priv, well);
2539
2540 mutex_unlock(&power_domains->lock);
2541
324513c0 2542 bxt_init_cdclk(dev_priv);
70c2c184
VS		 2543
2544 gen9_dbuf_enable(dev_priv);
2545
d7d7c9ee
ID		 2546 if (resume && dev_priv->csr.dmc_payload)
2547 intel_csr_load_program(dev_priv);
2548}
2549
2550void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
2551{
2552 struct i915_power_domains *power_domains = &dev_priv->power_domains;
2553 struct i915_power_well *well;
2554
2555 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2556
70c2c184
VS		 2557 gen9_dbuf_disable(dev_priv);
2558
324513c0 2559 bxt_uninit_cdclk(dev_priv);
d7d7c9ee
ID		 2560
2561 /* The spec doesn't call for removing the reset handshake flag */
2562
2563 /* Disable PG1 */
2564 mutex_lock(&power_domains->lock);
2565
2566 well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2567 intel_power_well_disable(dev_priv, well);
2568
2569 mutex_unlock(&power_domains->lock);
2570}
2571
70722468
VS		 2572static void chv_phy_control_init(struct drm_i915_private *dev_priv)
2573{
2574 struct i915_power_well *cmn_bc =
2575 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2576 struct i915_power_well *cmn_d =
2577 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
2578
2579 /*
2580 * DISPLAY_PHY_CONTROL can get corrupted if read. As a
2581 * workaround never ever read DISPLAY_PHY_CONTROL, and
2582 * instead maintain a shadow copy ourselves. Use the actual
e0fce78f
VS		 2583 * power well state and lane status to reconstruct the
2584 * expected initial value.
70722468
VS		 2585 */
2586 dev_priv->chv_phy_control =
bc284542
VS		 2587 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
2588 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
e0fce78f
VS		 2589 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
2590 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
2591 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
2592
2593 /*
2594 * If all lanes are disabled we leave the override disabled
2595 * with all power down bits cleared to match the state we
2596 * would use after disabling the port. Otherwise enable the
2597 * override and set the lane powerdown bits accding to the
2598 * current lane status.
2599 */
2600 if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
2601 uint32_t status = I915_READ(DPLL(PIPE_A));
2602 unsigned int mask;
2603
2604 mask = status & DPLL_PORTB_READY_MASK;
2605 if (mask == 0xf)
2606 mask = 0x0;
2607 else
2608 dev_priv->chv_phy_control |=
2609 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
2610
2611 dev_priv->chv_phy_control |=
2612 PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
2613
2614 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
2615 if (mask == 0xf)
2616 mask = 0x0;
2617 else
2618 dev_priv->chv_phy_control |=
2619 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
2620
2621 dev_priv->chv_phy_control |=
2622 PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
2623
70722468 2624 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
3be60de9
VS		 2625
2626 dev_priv->chv_phy_assert[DPIO_PHY0] = false;
2627 } else {
2628 dev_priv->chv_phy_assert[DPIO_PHY0] = true;
e0fce78f
VS		 2629 }
2630
2631 if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
2632 uint32_t status = I915_READ(DPIO_PHY_STATUS);
2633 unsigned int mask;
2634
2635 mask = status & DPLL_PORTD_READY_MASK;
2636
2637 if (mask == 0xf)
2638 mask = 0x0;
2639 else
2640 dev_priv->chv_phy_control |=
2641 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
2642
2643 dev_priv->chv_phy_control |=
2644 PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
2645
70722468 2646 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
3be60de9
VS		 2647
2648 dev_priv->chv_phy_assert[DPIO_PHY1] = false;
2649 } else {
2650 dev_priv->chv_phy_assert[DPIO_PHY1] = true;
e0fce78f
VS		 2651 }
2652
2653 I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
2654
2655 DRM_DEBUG_KMS("Initial PHY_CONTROL=0x%08x\n",
2656 dev_priv->chv_phy_control);
70722468
VS		 2657}
2658
9c065a7d
DV		 2659static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
2660{
2661 struct i915_power_well *cmn =
2662 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2663 struct i915_power_well *disp2d =
2664 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);
2665
9c065a7d 2666 /* If the display might be already active skip this */
5d93a6e5
VS		 2667 if (cmn->ops->is_enabled(dev_priv, cmn) &&
2668 disp2d->ops->is_enabled(dev_priv, disp2d) &&
9c065a7d
DV		 2669 I915_READ(DPIO_CTL) & DPIO_CMNRST)
2670 return;
2671
2672 DRM_DEBUG_KMS("toggling display PHY side reset\n");
2673
2674 /* cmnlane needs DPLL registers */
2675 disp2d->ops->enable(dev_priv, disp2d);
2676
2677 /*
2678 * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
2679 * Need to assert and de-assert PHY SB reset by gating the
2680 * common lane power, then un-gating it.
2681 * Simply ungating isn't enough to reset the PHY enough to get
2682 * ports and lanes running.
2683 */
2684 cmn->ops->disable(dev_priv, cmn);
2685}
2686
e4e7684f
DV		 2687/**
2688 * intel_power_domains_init_hw - initialize hardware power domain state
2689 * @dev_priv: i915 device instance
14bb2c11 2690 * @resume: Called from resume code paths or not
e4e7684f
DV		 2691 *
2692 * This function initializes the hardware power domain state and enables all
8d8c386c
ID		 2693 * power wells belonging to the INIT power domain. Power wells in other
2694 * domains (and not in the INIT domain) are referenced or disabled during the
2695 * modeset state HW readout. After that the reference count of each power well
2696 * must match its HW enabled state, see intel_power_domains_verify_state().
e4e7684f 2697 */
73dfc227 2698void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
9c065a7d 2699{
9c065a7d
DV		 2700 struct i915_power_domains *power_domains = &dev_priv->power_domains;
2701
2702 power_domains->initializing = true;
2703
b976dc53 2704 if (IS_GEN9_BC(dev_priv)) {
73dfc227 2705 skl_display_core_init(dev_priv, resume);
b817c440 2706 } else if (IS_GEN9_LP(dev_priv)) {
d7d7c9ee 2707 bxt_display_core_init(dev_priv, resume);
920a14b2 2708 } else if (IS_CHERRYVIEW(dev_priv)) {
770effb1 2709 mutex_lock(&power_domains->lock);
70722468 2710 chv_phy_control_init(dev_priv);
770effb1 2711 mutex_unlock(&power_domains->lock);
11a914c2 2712 } else if (IS_VALLEYVIEW(dev_priv)) {
9c065a7d
DV		 2713 mutex_lock(&power_domains->lock);
2714 vlv_cmnlane_wa(dev_priv);
2715 mutex_unlock(&power_domains->lock);
2716 }
2717
2718 /* For now, we need the power well to be always enabled. */
2719 intel_display_set_init_power(dev_priv, true);
d314cd43
ID		 2720 /* Disable power support if the user asked so. */
2721 if (!i915.disable_power_well)
2722 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
30eade12 2723 intel_power_domains_sync_hw(dev_priv);
9c065a7d
DV		 2724 power_domains->initializing = false;
2725}
2726
73dfc227
ID		 2727/**
2728 * intel_power_domains_suspend - suspend power domain state
2729 * @dev_priv: i915 device instance
2730 *
2731 * This function prepares the hardware power domain state before entering
2732 * system suspend. It must be paired with intel_power_domains_init_hw().
2733 */
2734void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
2735{
d314cd43
ID		 2736 /*
2737 * Even if power well support was disabled we still want to disable
2738 * power wells while we are system suspended.
2739 */
2740 if (!i915.disable_power_well)
2741 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
2622d79b 2742
b976dc53 2743 if (IS_GEN9_BC(dev_priv))
2622d79b 2744 skl_display_core_uninit(dev_priv);
b817c440 2745 else if (IS_GEN9_LP(dev_priv))
d7d7c9ee 2746 bxt_display_core_uninit(dev_priv);
73dfc227
ID		 2747}
2748
8d8c386c
ID		 2749static void intel_power_domains_dump_info(struct drm_i915_private *dev_priv)
2750{
2751 struct i915_power_domains *power_domains = &dev_priv->power_domains;
2752 struct i915_power_well *power_well;
2753
2754 for_each_power_well(dev_priv, power_well) {
2755 enum intel_display_power_domain domain;
2756
2757 DRM_DEBUG_DRIVER("%-25s %d\n",
2758 power_well->name, power_well->count);
2759
2760 for_each_power_domain(domain, power_well->domains)
2761 DRM_DEBUG_DRIVER(" %-23s %d\n",
2762 intel_display_power_domain_str(domain),
2763 power_domains->domain_use_count[domain]);
2764 }
2765}
2766
2767/**
2768 * intel_power_domains_verify_state - verify the HW/SW state for all power wells
2769 * @dev_priv: i915 device instance
2770 *
2771 * Verify if the reference count of each power well matches its HW enabled
2772 * state and the total refcount of the domains it belongs to. This must be
2773 * called after modeset HW state sanitization, which is responsible for
2774 * acquiring reference counts for any power wells in use and disabling the
2775 * ones left on by BIOS but not required by any active output.
2776 */
2777void intel_power_domains_verify_state(struct drm_i915_private *dev_priv)
2778{
2779 struct i915_power_domains *power_domains = &dev_priv->power_domains;
2780 struct i915_power_well *power_well;
2781 bool dump_domain_info;
2782
2783 mutex_lock(&power_domains->lock);
2784
2785 dump_domain_info = false;
2786 for_each_power_well(dev_priv, power_well) {
2787 enum intel_display_power_domain domain;
2788 int domains_count;
2789 bool enabled;
2790
2791 /*
2792 * Power wells not belonging to any domain (like the MISC_IO
2793 * and PW1 power wells) are under FW control, so ignore them,
2794 * since their state can change asynchronously.
2795 */
2796 if (!power_well->domains)
2797 continue;
2798
2799 enabled = power_well->ops->is_enabled(dev_priv, power_well);
2800 if ((power_well->count || power_well->always_on) != enabled)
2801 DRM_ERROR("power well %s state mismatch (refcount %d/enabled %d)",
2802 power_well->name, power_well->count, enabled);
2803
2804 domains_count = 0;
2805 for_each_power_domain(domain, power_well->domains)
2806 domains_count += power_domains->domain_use_count[domain];
2807
2808 if (power_well->count != domains_count) {
2809 DRM_ERROR("power well %s refcount/domain refcount mismatch "
2810 "(refcount %d/domains refcount %d)\n",
2811 power_well->name, power_well->count,
2812 domains_count);
2813 dump_domain_info = true;
2814 }
2815 }
2816
2817 if (dump_domain_info) {
2818 static bool dumped;
2819
2820 if (!dumped) {
2821 intel_power_domains_dump_info(dev_priv);
2822 dumped = true;
2823 }
2824 }
2825
2826 mutex_unlock(&power_domains->lock);
2827}
2828
e4e7684f
DV		 2829/**
2830 * intel_runtime_pm_get - grab a runtime pm reference
2831 * @dev_priv: i915 device instance
2832 *
2833 * This function grabs a device-level runtime pm reference (mostly used for GEM
2834 * code to ensure the GTT or GT is on) and ensures that it is powered up.
2835 *
2836 * Any runtime pm reference obtained by this function must have a symmetric
2837 * call to intel_runtime_pm_put() to release the reference again.
2838 */
9c065a7d
DV		 2839void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
2840{
52a05c30
DW		 2841 struct pci_dev *pdev = dev_priv->drm.pdev;
2842 struct device *kdev = &pdev->dev;
f5073824 2843 int ret;
9c065a7d 2844
f5073824
ID		 2845 ret = pm_runtime_get_sync(kdev);
2846 WARN_ONCE(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
1f814dac
ID		 2847
2848 atomic_inc(&dev_priv->pm.wakeref_count);
c9b8846a 2849 assert_rpm_wakelock_held(dev_priv);
9c065a7d
DV		 2850}
2851
09731280
ID		 2852/**
2853 * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use
2854 * @dev_priv: i915 device instance
2855 *
2856 * This function grabs a device-level runtime pm reference if the device is
2857 * already in use and ensures that it is powered up.
2858 *
2859 * Any runtime pm reference obtained by this function must have a symmetric
2860 * call to intel_runtime_pm_put() to release the reference again.
2861 */
2862bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
2863{
52a05c30
DW		 2864 struct pci_dev *pdev = dev_priv->drm.pdev;
2865 struct device *kdev = &pdev->dev;
09731280 2866
135dc79e 2867 if (IS_ENABLED(CONFIG_PM)) {
c49d13ee 2868 int ret = pm_runtime_get_if_in_use(kdev);
09731280 2869
135dc79e
CW		 2870 /*
2871 * In cases runtime PM is disabled by the RPM core and we get
2872 * an -EINVAL return value we are not supposed to call this
2873 * function, since the power state is undefined. This applies
2874 * atm to the late/early system suspend/resume handlers.
2875 */
f5073824
ID		 2876 WARN_ONCE(ret < 0,
2877 "pm_runtime_get_if_in_use() failed: %d\n", ret);
135dc79e
CW		 2878 if (ret <= 0)
2879 return false;
2880 }
09731280
ID		 2881
2882 atomic_inc(&dev_priv->pm.wakeref_count);
2883 assert_rpm_wakelock_held(dev_priv);
2884
2885 return true;
2886}
2887
e4e7684f
DV		 2888/**
2889 * intel_runtime_pm_get_noresume - grab a runtime pm reference
2890 * @dev_priv: i915 device instance
2891 *
2892 * This function grabs a device-level runtime pm reference (mostly used for GEM
2893 * code to ensure the GTT or GT is on).
2894 *
2895 * It will _not_ power up the device but instead only check that it's powered
2896 * on. Therefore it is only valid to call this functions from contexts where
2897 * the device is known to be powered up and where trying to power it up would
2898 * result in hilarity and deadlocks. That pretty much means only the system
2899 * suspend/resume code where this is used to grab runtime pm references for
2900 * delayed setup down in work items.
2901 *
2902 * Any runtime pm reference obtained by this function must have a symmetric
2903 * call to intel_runtime_pm_put() to release the reference again.
2904 */
9c065a7d
DV		 2905void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
2906{
52a05c30
DW		 2907 struct pci_dev *pdev = dev_priv->drm.pdev;
2908 struct device *kdev = &pdev->dev;
9c065a7d 2909
c9b8846a 2910 assert_rpm_wakelock_held(dev_priv);
c49d13ee 2911 pm_runtime_get_noresume(kdev);
1f814dac
ID		 2912
2913 atomic_inc(&dev_priv->pm.wakeref_count);
9c065a7d
DV		 2914}
2915
e4e7684f
DV		 2916/**
2917 * intel_runtime_pm_put - release a runtime pm reference
2918 * @dev_priv: i915 device instance
2919 *
2920 * This function drops the device-level runtime pm reference obtained by
2921 * intel_runtime_pm_get() and might power down the corresponding
2922 * hardware block right away if this is the last reference.
2923 */
9c065a7d
DV		 2924void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
2925{
52a05c30
DW		 2926 struct pci_dev *pdev = dev_priv->drm.pdev;
2927 struct device *kdev = &pdev->dev;
9c065a7d 2928
542db3cd 2929 assert_rpm_wakelock_held(dev_priv);
2eedfc7d 2930 atomic_dec(&dev_priv->pm.wakeref_count);
1f814dac 2931
c49d13ee
DW		 2932 pm_runtime_mark_last_busy(kdev);
2933 pm_runtime_put_autosuspend(kdev);
9c065a7d
DV		 2934}
2935
e4e7684f
DV		 2936/**
2937 * intel_runtime_pm_enable - enable runtime pm
2938 * @dev_priv: i915 device instance
2939 *
2940 * This function enables runtime pm at the end of the driver load sequence.
2941 *
2942 * Note that this function does currently not enable runtime pm for the
2943 * subordinate display power domains. That is only done on the first modeset
2944 * using intel_display_set_init_power().
2945 */
f458ebbc 2946void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
9c065a7d 2947{
52a05c30 2948 struct pci_dev *pdev = dev_priv->drm.pdev;
52a05c30 2949 struct device *kdev = &pdev->dev;
9c065a7d 2950
c49d13ee
DW		 2951 pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
2952 pm_runtime_mark_last_busy(kdev);
cbc68dc9 2953
25b181b4
ID		 2954 /*
2955 * Take a permanent reference to disable the RPM functionality and drop
2956 * it only when unloading the driver. Use the low level get/put helpers,
2957 * so the driver's own RPM reference tracking asserts also work on
2958 * platforms without RPM support.
2959 */
6772ffe0 2960 if (!HAS_RUNTIME_PM(dev_priv)) {
f5073824
ID		 2961 int ret;
2962
c49d13ee 2963 pm_runtime_dont_use_autosuspend(kdev);
f5073824
ID		 2964 ret = pm_runtime_get_sync(kdev);
2965 WARN(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
cbc68dc9 2966 } else {
c49d13ee 2967 pm_runtime_use_autosuspend(kdev);
cbc68dc9 2968 }
9c065a7d 2969
aabee1bb
ID		 2970 /*
2971 * The core calls the driver load handler with an RPM reference held.
2972 * We drop that here and will reacquire it during unloading in
2973 * intel_power_domains_fini().
2974 */
c49d13ee 2975 pm_runtime_put_autosuspend(kdev);
9c065a7d 2976}
Ubuntu Artful kernel mirror