]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/gpu/drm/i915/i915_drv.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Split GGTT initialisation between probing and setup
[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / i915 / i915_drv.c
CommitLineData
1da177e4
LT		 1/* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
2 */
0d6aa60b 3/*
bc54fd1a 4 *
1da177e4
LT		 5 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
6 * All Rights Reserved.
bc54fd1a
DA		 7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
23 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
0d6aa60b 28 */
1da177e4 29
e5747e3a 30#include <linux/acpi.h>
0673ad47
CW		 31#include <linux/device.h>
32#include <linux/oom.h>
e0cd3608 33#include <linux/module.h>
0673ad47
CW		 34#include <linux/pci.h>
35#include <linux/pm.h>
d6102977 36#include <linux/pm_runtime.h>
0673ad47
CW		 37#include <linux/pnp.h>
38#include <linux/slab.h>
39#include <linux/vgaarb.h>
704ab614 40#include <linux/vga_switcheroo.h>
0673ad47
CW		 41#include <linux/vt.h>
42#include <acpi/video.h>
43
44#include <drm/drmP.h>
760285e7 45#include <drm/drm_crtc_helper.h>
0673ad47
CW		 46#include <drm/i915_drm.h>
47
48#include "i915_drv.h"
49#include "i915_trace.h"
50#include "i915_vgpu.h"
51#include "intel_drv.h"
79e53945 52
112b715e
KH		 53static struct drm_driver driver;
54
0673ad47
CW		 55static unsigned int i915_load_fail_count;
56
57bool __i915_inject_load_failure(const char *func, int line)
58{
59 if (i915_load_fail_count >= i915.inject_load_failure)
60 return false;
61
62 if (++i915_load_fail_count == i915.inject_load_failure) {
63 DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n",
64 i915.inject_load_failure, func, line);
65 return true;
66 }
67
68 return false;
69}
70
71#define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI"
72#define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \
73 "providing the dmesg log by booting with drm.debug=0xf"
74
75void
76__i915_printk(struct drm_i915_private *dev_priv, const char *level,
77 const char *fmt, ...)
78{
79 static bool shown_bug_once;
fb939420 80 struct device *dev = dev_priv->drm.dev;
0673ad47
CW		 81 bool is_error = level[1] <= KERN_ERR[1];
82 bool is_debug = level[1] == KERN_DEBUG[1];
83 struct va_format vaf;
84 va_list args;
85
86 if (is_debug && !(drm_debug & DRM_UT_DRIVER))
87 return;
88
89 va_start(args, fmt);
90
91 vaf.fmt = fmt;
92 vaf.va = &args;
93
94 dev_printk(level, dev, "[" DRM_NAME ":%ps] %pV",
95 __builtin_return_address(0), &vaf);
96
97 if (is_error && !shown_bug_once) {
98 dev_notice(dev, "%s", FDO_BUG_MSG);
99 shown_bug_once = true;
100 }
101
102 va_end(args);
103}
104
105static bool i915_error_injected(struct drm_i915_private *dev_priv)
106{
107 return i915.inject_load_failure &&
108 i915_load_fail_count == i915.inject_load_failure;
109}
110
111#define i915_load_error(dev_priv, fmt, ...) \
112 __i915_printk(dev_priv, \
113 i915_error_injected(dev_priv) ? KERN_DEBUG : KERN_ERR, \
114 fmt, ##__VA_ARGS__)
115
116
117static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
118{
119 enum intel_pch ret = PCH_NOP;
120
121 /*
122 * In a virtualized passthrough environment we can be in a
123 * setup where the ISA bridge is not able to be passed through.
124 * In this case, a south bridge can be emulated and we have to
125 * make an educated guess as to which PCH is really there.
126 */
127
128 if (IS_GEN5(dev)) {
129 ret = PCH_IBX;
130 DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n");
131 } else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
132 ret = PCH_CPT;
133 DRM_DEBUG_KMS("Assuming CouarPoint PCH\n");
134 } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
135 ret = PCH_LPT;
136 DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
137 } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
138 ret = PCH_SPT;
139 DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
140 }
141
142 return ret;
143}
144
145static void intel_detect_pch(struct drm_device *dev)
146{
fac5e23e 147 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 148 struct pci_dev *pch = NULL;
149
150 /* In all current cases, num_pipes is equivalent to the PCH_NOP setting
151 * (which really amounts to a PCH but no South Display).
152 */
153 if (INTEL_INFO(dev)->num_pipes == 0) {
154 dev_priv->pch_type = PCH_NOP;
155 return;
156 }
157
158 /*
159 * The reason to probe ISA bridge instead of Dev31:Fun0 is to
160 * make graphics device passthrough work easy for VMM, that only
161 * need to expose ISA bridge to let driver know the real hardware
162 * underneath. This is a requirement from virtualization team.
163 *
164 * In some virtualized environments (e.g. XEN), there is irrelevant
165 * ISA bridge in the system. To work reliably, we should scan trhough
166 * all the ISA bridge devices and check for the first match, instead
167 * of only checking the first one.
168 */
169 while ((pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, pch))) {
170 if (pch->vendor == PCI_VENDOR_ID_INTEL) {
171 unsigned short id = pch->device & INTEL_PCH_DEVICE_ID_MASK;
172 dev_priv->pch_id = id;
173
174 if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
175 dev_priv->pch_type = PCH_IBX;
176 DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
177 WARN_ON(!IS_GEN5(dev));
178 } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
179 dev_priv->pch_type = PCH_CPT;
180 DRM_DEBUG_KMS("Found CougarPoint PCH\n");
181 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
182 } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
183 /* PantherPoint is CPT compatible */
184 dev_priv->pch_type = PCH_CPT;
185 DRM_DEBUG_KMS("Found PantherPoint PCH\n");
186 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
187 } else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
188 dev_priv->pch_type = PCH_LPT;
189 DRM_DEBUG_KMS("Found LynxPoint PCH\n");
190 WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
191 WARN_ON(IS_HSW_ULT(dev) || IS_BDW_ULT(dev));
192 } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
193 dev_priv->pch_type = PCH_LPT;
194 DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
195 WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
196 WARN_ON(!IS_HSW_ULT(dev) && !IS_BDW_ULT(dev));
197 } else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
198 dev_priv->pch_type = PCH_SPT;
199 DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
200 WARN_ON(!IS_SKYLAKE(dev) &&
201 !IS_KABYLAKE(dev));
202 } else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
203 dev_priv->pch_type = PCH_SPT;
204 DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
205 WARN_ON(!IS_SKYLAKE(dev) &&
206 !IS_KABYLAKE(dev));
22dea0be
RV		 207 } else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) {
208 dev_priv->pch_type = PCH_KBP;
209 DRM_DEBUG_KMS("Found KabyPoint PCH\n");
210 WARN_ON(!IS_KABYLAKE(dev));
0673ad47
CW		 211 } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
212 (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) ||
213 ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) &&
214 pch->subsystem_vendor ==
215 PCI_SUBVENDOR_ID_REDHAT_QUMRANET &&
216 pch->subsystem_device ==
217 PCI_SUBDEVICE_ID_QEMU)) {
218 dev_priv->pch_type = intel_virt_detect_pch(dev);
219 } else
220 continue;
221
222 break;
223 }
224 }
225 if (!pch)
226 DRM_DEBUG_KMS("No PCH found.\n");
227
228 pci_dev_put(pch);
229}
230
0673ad47
CW		 231static int i915_getparam(struct drm_device *dev, void *data,
232 struct drm_file *file_priv)
233{
fac5e23e 234 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 235 drm_i915_getparam_t *param = data;
236 int value;
237
238 switch (param->param) {
239 case I915_PARAM_IRQ_ACTIVE:
240 case I915_PARAM_ALLOW_BATCHBUFFER:
241 case I915_PARAM_LAST_DISPATCH:
242 /* Reject all old ums/dri params. */
243 return -ENODEV;
244 case I915_PARAM_CHIPSET_ID:
245 value = dev->pdev->device;
246 break;
247 case I915_PARAM_REVISION:
248 value = dev->pdev->revision;
249 break;
250 case I915_PARAM_HAS_GEM:
251 value = 1;
252 break;
253 case I915_PARAM_NUM_FENCES_AVAIL:
254 value = dev_priv->num_fence_regs;
255 break;
256 case I915_PARAM_HAS_OVERLAY:
257 value = dev_priv->overlay ? 1 : 0;
258 break;
259 case I915_PARAM_HAS_PAGEFLIPPING:
260 value = 1;
261 break;
262 case I915_PARAM_HAS_EXECBUF2:
263 /* depends on GEM */
264 value = 1;
265 break;
266 case I915_PARAM_HAS_BSD:
267 value = intel_engine_initialized(&dev_priv->engine[VCS]);
268 break;
269 case I915_PARAM_HAS_BLT:
270 value = intel_engine_initialized(&dev_priv->engine[BCS]);
271 break;
272 case I915_PARAM_HAS_VEBOX:
273 value = intel_engine_initialized(&dev_priv->engine[VECS]);
274 break;
275 case I915_PARAM_HAS_BSD2:
276 value = intel_engine_initialized(&dev_priv->engine[VCS2]);
277 break;
278 case I915_PARAM_HAS_RELAXED_FENCING:
279 value = 1;
280 break;
281 case I915_PARAM_HAS_COHERENT_RINGS:
282 value = 1;
283 break;
284 case I915_PARAM_HAS_EXEC_CONSTANTS:
285 value = INTEL_INFO(dev)->gen >= 4;
286 break;
287 case I915_PARAM_HAS_RELAXED_DELTA:
288 value = 1;
289 break;
290 case I915_PARAM_HAS_GEN7_SOL_RESET:
291 value = 1;
292 break;
293 case I915_PARAM_HAS_LLC:
294 value = HAS_LLC(dev);
295 break;
296 case I915_PARAM_HAS_WT:
297 value = HAS_WT(dev);
298 break;
299 case I915_PARAM_HAS_ALIASING_PPGTT:
300 value = USES_PPGTT(dev);
301 break;
302 case I915_PARAM_HAS_WAIT_TIMEOUT:
303 value = 1;
304 break;
305 case I915_PARAM_HAS_SEMAPHORES:
39df9190 306 value = i915.semaphores;
0673ad47
CW		 307 break;
308 case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
309 value = 1;
310 break;
311 case I915_PARAM_HAS_SECURE_BATCHES:
312 value = capable(CAP_SYS_ADMIN);
313 break;
314 case I915_PARAM_HAS_PINNED_BATCHES:
315 value = 1;
316 break;
317 case I915_PARAM_HAS_EXEC_NO_RELOC:
318 value = 1;
319 break;
320 case I915_PARAM_HAS_EXEC_HANDLE_LUT:
321 value = 1;
322 break;
323 case I915_PARAM_CMD_PARSER_VERSION:
324 value = i915_cmd_parser_get_version(dev_priv);
325 break;
326 case I915_PARAM_HAS_COHERENT_PHYS_GTT:
327 value = 1;
328 break;
329 case I915_PARAM_MMAP_VERSION:
330 value = 1;
331 break;
332 case I915_PARAM_SUBSLICE_TOTAL:
333 value = INTEL_INFO(dev)->subslice_total;
334 if (!value)
335 return -ENODEV;
336 break;
337 case I915_PARAM_EU_TOTAL:
338 value = INTEL_INFO(dev)->eu_total;
339 if (!value)
340 return -ENODEV;
341 break;
342 case I915_PARAM_HAS_GPU_RESET:
343 value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv);
344 break;
345 case I915_PARAM_HAS_RESOURCE_STREAMER:
346 value = HAS_RESOURCE_STREAMER(dev);
347 break;
348 case I915_PARAM_HAS_EXEC_SOFTPIN:
349 value = 1;
350 break;
37f501af 351 case I915_PARAM_HAS_POOLED_EU:
352 value = HAS_POOLED_EU(dev);
353 break;
354 case I915_PARAM_MIN_EU_IN_POOL:
355 value = INTEL_INFO(dev)->min_eu_in_pool;
356 break;
0673ad47
CW		 357 default:
358 DRM_DEBUG("Unknown parameter %d\n", param->param);
359 return -EINVAL;
360 }
361
dda33009 362 if (put_user(value, param->value))
0673ad47 363 return -EFAULT;
0673ad47
CW		 364
365 return 0;
366}
367
368static int i915_get_bridge_dev(struct drm_device *dev)
369{
fac5e23e 370 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 371
372 dev_priv->bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
373 if (!dev_priv->bridge_dev) {
374 DRM_ERROR("bridge device not found\n");
375 return -1;
376 }
377 return 0;
378}
379
380/* Allocate space for the MCH regs if needed, return nonzero on error */
381static int
382intel_alloc_mchbar_resource(struct drm_device *dev)
383{
fac5e23e 384 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 385 int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
386 u32 temp_lo, temp_hi = 0;
387 u64 mchbar_addr;
388 int ret;
389
390 if (INTEL_INFO(dev)->gen >= 4)
391 pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
392 pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
393 mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
394
395 /* If ACPI doesn't have it, assume we need to allocate it ourselves */
396#ifdef CONFIG_PNP
397 if (mchbar_addr &&
398 pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
399 return 0;
400#endif
401
402 /* Get some space for it */
403 dev_priv->mch_res.name = "i915 MCHBAR";
404 dev_priv->mch_res.flags = IORESOURCE_MEM;
405 ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,
406 &dev_priv->mch_res,
407 MCHBAR_SIZE, MCHBAR_SIZE,
408 PCIBIOS_MIN_MEM,
409 0, pcibios_align_resource,
410 dev_priv->bridge_dev);
411 if (ret) {
412 DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret);
413 dev_priv->mch_res.start = 0;
414 return ret;
415 }
416
417 if (INTEL_INFO(dev)->gen >= 4)
418 pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
419 upper_32_bits(dev_priv->mch_res.start));
420
421 pci_write_config_dword(dev_priv->bridge_dev, reg,
422 lower_32_bits(dev_priv->mch_res.start));
423 return 0;
424}
425
426/* Setup MCHBAR if possible, return true if we should disable it again */
427static void
428intel_setup_mchbar(struct drm_device *dev)
429{
fac5e23e 430 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 431 int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
432 u32 temp;
433 bool enabled;
434
435 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
436 return;
437
438 dev_priv->mchbar_need_disable = false;
439
440 if (IS_I915G(dev) || IS_I915GM(dev)) {
441 pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);
442 enabled = !!(temp & DEVEN_MCHBAR_EN);
443 } else {
444 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
445 enabled = temp & 1;
446 }
447
448 /* If it's already enabled, don't have to do anything */
449 if (enabled)
450 return;
451
452 if (intel_alloc_mchbar_resource(dev))
453 return;
454
455 dev_priv->mchbar_need_disable = true;
456
457 /* Space is allocated or reserved, so enable it. */
458 if (IS_I915G(dev) || IS_I915GM(dev)) {
459 pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
460 temp | DEVEN_MCHBAR_EN);
461 } else {
462 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
463 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);
464 }
465}
466
467static void
468intel_teardown_mchbar(struct drm_device *dev)
469{
fac5e23e 470 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 471 int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
472
473 if (dev_priv->mchbar_need_disable) {
474 if (IS_I915G(dev) || IS_I915GM(dev)) {
475 u32 deven_val;
476
477 pci_read_config_dword(dev_priv->bridge_dev, DEVEN,
478 &deven_val);
479 deven_val &= ~DEVEN_MCHBAR_EN;
480 pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
481 deven_val);
482 } else {
483 u32 mchbar_val;
484
485 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg,
486 &mchbar_val);
487 mchbar_val &= ~1;
488 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg,
489 mchbar_val);
490 }
491 }
492
493 if (dev_priv->mch_res.start)
494 release_resource(&dev_priv->mch_res);
495}
496
497/* true = enable decode, false = disable decoder */
498static unsigned int i915_vga_set_decode(void *cookie, bool state)
499{
500 struct drm_device *dev = cookie;
501
502 intel_modeset_vga_set_state(dev, state);
503 if (state)
504 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
505 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
506 else
507 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
508}
509
510static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
511{
512 struct drm_device *dev = pci_get_drvdata(pdev);
513 pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
514
515 if (state == VGA_SWITCHEROO_ON) {
516 pr_info("switched on\n");
517 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
518 /* i915 resume handler doesn't set to D0 */
519 pci_set_power_state(dev->pdev, PCI_D0);
520 i915_resume_switcheroo(dev);
521 dev->switch_power_state = DRM_SWITCH_POWER_ON;
522 } else {
523 pr_info("switched off\n");
524 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
525 i915_suspend_switcheroo(dev, pmm);
526 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
527 }
528}
529
530static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
531{
532 struct drm_device *dev = pci_get_drvdata(pdev);
533
534 /*
535 * FIXME: open_count is protected by drm_global_mutex but that would lead to
536 * locking inversion with the driver load path. And the access here is
537 * completely racy anyway. So don't bother with locking for now.
538 */
539 return dev->open_count == 0;
540}
541
542static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
543 .set_gpu_state = i915_switcheroo_set_state,
544 .reprobe = NULL,
545 .can_switch = i915_switcheroo_can_switch,
546};
547
548static void i915_gem_fini(struct drm_device *dev)
549{
550 struct drm_i915_private *dev_priv = to_i915(dev);
551
552 /*
553 * Neither the BIOS, ourselves or any other kernel
554 * expects the system to be in execlists mode on startup,
555 * so we need to reset the GPU back to legacy mode. And the only
556 * known way to disable logical contexts is through a GPU reset.
557 *
558 * So in order to leave the system in a known default configuration,
559 * always reset the GPU upon unload. Afterwards we then clean up the
560 * GEM state tracking, flushing off the requests and leaving the
561 * system in a known idle state.
562 *
563 * Note that is of the upmost importance that the GPU is idle and
564 * all stray writes are flushed *before* we dismantle the backing
565 * storage for the pinned objects.
566 *
567 * However, since we are uncertain that reseting the GPU on older
568 * machines is a good idea, we don't - just in case it leaves the
569 * machine in an unusable condition.
570 */
571 if (HAS_HW_CONTEXTS(dev)) {
572 int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
573 WARN_ON(reset && reset != -ENODEV);
574 }
575
576 mutex_lock(&dev->struct_mutex);
577 i915_gem_reset(dev);
578 i915_gem_cleanup_engines(dev);
579 i915_gem_context_fini(dev);
580 mutex_unlock(&dev->struct_mutex);
581
582 WARN_ON(!list_empty(&to_i915(dev)->context_list));
583}
584
585static int i915_load_modeset_init(struct drm_device *dev)
586{
fac5e23e 587 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 588 int ret;
589
590 if (i915_inject_load_failure())
591 return -ENODEV;
592
593 ret = intel_bios_init(dev_priv);
594 if (ret)
595 DRM_INFO("failed to find VBIOS tables\n");
596
597 /* If we have > 1 VGA cards, then we need to arbitrate access
598 * to the common VGA resources.
599 *
600 * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
601 * then we do not take part in VGA arbitration and the
602 * vga_client_register() fails with -ENODEV.
603 */
604 ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode);
605 if (ret && ret != -ENODEV)
606 goto out;
607
608 intel_register_dsm_handler();
609
610 ret = vga_switcheroo_register_client(dev->pdev, &i915_switcheroo_ops, false);
611 if (ret)
612 goto cleanup_vga_client;
613
614 /* must happen before intel_power_domains_init_hw() on VLV/CHV */
615 intel_update_rawclk(dev_priv);
616
617 intel_power_domains_init_hw(dev_priv, false);
618
619 intel_csr_ucode_init(dev_priv);
620
621 ret = intel_irq_install(dev_priv);
622 if (ret)
623 goto cleanup_csr;
624
625 intel_setup_gmbus(dev);
626
627 /* Important: The output setup functions called by modeset_init need
628 * working irqs for e.g. gmbus and dp aux transfers. */
629 intel_modeset_init(dev);
630
631 intel_guc_init(dev);
632
633 ret = i915_gem_init(dev);
634 if (ret)
635 goto cleanup_irq;
636
637 intel_modeset_gem_init(dev);
638
639 if (INTEL_INFO(dev)->num_pipes == 0)
640 return 0;
641
642 ret = intel_fbdev_init(dev);
643 if (ret)
644 goto cleanup_gem;
645
646 /* Only enable hotplug handling once the fbdev is fully set up. */
647 intel_hpd_init(dev_priv);
648
649 drm_kms_helper_poll_init(dev);
650
651 return 0;
652
653cleanup_gem:
654 i915_gem_fini(dev);
655cleanup_irq:
656 intel_guc_fini(dev);
657 drm_irq_uninstall(dev);
658 intel_teardown_gmbus(dev);
659cleanup_csr:
660 intel_csr_ucode_fini(dev_priv);
661 intel_power_domains_fini(dev_priv);
662 vga_switcheroo_unregister_client(dev->pdev);
663cleanup_vga_client:
664 vga_client_register(dev->pdev, NULL, NULL, NULL);
665out:
666 return ret;
667}
668
669#if IS_ENABLED(CONFIG_FB)
670static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
671{
672 struct apertures_struct *ap;
91c8a326 673 struct pci_dev *pdev = dev_priv->drm.pdev;
0673ad47
CW		 674 struct i915_ggtt *ggtt = &dev_priv->ggtt;
675 bool primary;
676 int ret;
677
678 ap = alloc_apertures(1);
679 if (!ap)
680 return -ENOMEM;
681
682 ap->ranges[0].base = ggtt->mappable_base;
683 ap->ranges[0].size = ggtt->mappable_end;
684
685 primary =
686 pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
687
688 ret = remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
689
690 kfree(ap);
691
692 return ret;
693}
694#else
695static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
696{
697 return 0;
698}
699#endif
700
701#if !defined(CONFIG_VGA_CONSOLE)
702static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
703{
704 return 0;
705}
706#elif !defined(CONFIG_DUMMY_CONSOLE)
707static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
708{
709 return -ENODEV;
710}
711#else
712static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
713{
714 int ret = 0;
715
716 DRM_INFO("Replacing VGA console driver\n");
717
718 console_lock();
719 if (con_is_bound(&vga_con))
720 ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1);
721 if (ret == 0) {
722 ret = do_unregister_con_driver(&vga_con);
723
724 /* Ignore "already unregistered". */
725 if (ret == -ENODEV)
726 ret = 0;
727 }
728 console_unlock();
729
730 return ret;
731}
732#endif
733
0673ad47
CW		 734static void intel_init_dpio(struct drm_i915_private *dev_priv)
735{
736 /*
737 * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
738 * CHV x1 PHY (DP/HDMI D)
739 * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
740 */
741 if (IS_CHERRYVIEW(dev_priv)) {
742 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
743 DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
744 } else if (IS_VALLEYVIEW(dev_priv)) {
745 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
746 }
747}
748
749static int i915_workqueues_init(struct drm_i915_private *dev_priv)
750{
751 /*
752 * The i915 workqueue is primarily used for batched retirement of
753 * requests (and thus managing bo) once the task has been completed
754 * by the GPU. i915_gem_retire_requests() is called directly when we
755 * need high-priority retirement, such as waiting for an explicit
756 * bo.
757 *
758 * It is also used for periodic low-priority events, such as
759 * idle-timers and recording error state.
760 *
761 * All tasks on the workqueue are expected to acquire the dev mutex
762 * so there is no point in running more than one instance of the
763 * workqueue at any time. Use an ordered one.
764 */
765 dev_priv->wq = alloc_ordered_workqueue("i915", 0);
766 if (dev_priv->wq == NULL)
767 goto out_err;
768
769 dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
770 if (dev_priv->hotplug.dp_wq == NULL)
771 goto out_free_wq;
772
0673ad47
CW		 773 return 0;
774
0673ad47
CW		 775out_free_wq:
776 destroy_workqueue(dev_priv->wq);
777out_err:
778 DRM_ERROR("Failed to allocate workqueues.\n");
779
780 return -ENOMEM;
781}
782
783static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
784{
0673ad47
CW		 785 destroy_workqueue(dev_priv->hotplug.dp_wq);
786 destroy_workqueue(dev_priv->wq);
787}
788
789/**
790 * i915_driver_init_early - setup state not requiring device access
791 * @dev_priv: device private
792 *
793 * Initialize everything that is a "SW-only" state, that is state not
794 * requiring accessing the device or exposing the driver via kernel internal
795 * or userspace interfaces. Example steps belonging here: lock initialization,
796 * system memory allocation, setting up device specific attributes and
797 * function hooks not requiring accessing the device.
798 */
799static int i915_driver_init_early(struct drm_i915_private *dev_priv,
800 const struct pci_device_id *ent)
801{
802 const struct intel_device_info *match_info =
803 (struct intel_device_info *)ent->driver_data;
804 struct intel_device_info *device_info;
805 int ret = 0;
806
807 if (i915_inject_load_failure())
808 return -ENODEV;
809
810 /* Setup the write-once "constant" device info */
94b4f3ba 811 device_info = mkwrite_device_info(dev_priv);
0673ad47
CW		 812 memcpy(device_info, match_info, sizeof(*device_info));
813 device_info->device_id = dev_priv->drm.pdev->device;
814
815 BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE);
816 device_info->gen_mask = BIT(device_info->gen - 1);
817
818 spin_lock_init(&dev_priv->irq_lock);
819 spin_lock_init(&dev_priv->gpu_error.lock);
820 mutex_init(&dev_priv->backlight_lock);
821 spin_lock_init(&dev_priv->uncore.lock);
822 spin_lock_init(&dev_priv->mm.object_stat_lock);
823 spin_lock_init(&dev_priv->mmio_flip_lock);
824 mutex_init(&dev_priv->sb_lock);
825 mutex_init(&dev_priv->modeset_restore_lock);
826 mutex_init(&dev_priv->av_mutex);
827 mutex_init(&dev_priv->wm.wm_mutex);
828 mutex_init(&dev_priv->pps_mutex);
829
830 ret = i915_workqueues_init(dev_priv);
831 if (ret < 0)
832 return ret;
833
834 ret = intel_gvt_init(dev_priv);
835 if (ret < 0)
836 goto err_workqueues;
837
838 /* This must be called before any calls to HAS_PCH_* */
839 intel_detect_pch(&dev_priv->drm);
840
841 intel_pm_setup(&dev_priv->drm);
842 intel_init_dpio(dev_priv);
843 intel_power_domains_init(dev_priv);
844 intel_irq_init(dev_priv);
845 intel_init_display_hooks(dev_priv);
846 intel_init_clock_gating_hooks(dev_priv);
847 intel_init_audio_hooks(dev_priv);
848 i915_gem_load_init(&dev_priv->drm);
849
850 intel_display_crc_init(&dev_priv->drm);
851
94b4f3ba 852 intel_device_info_dump(dev_priv);
0673ad47
CW		 853
854 /* Not all pre-production machines fall into this category, only the
855 * very first ones. Almost everything should work, except for maybe
856 * suspend/resume. And we don't implement workarounds that affect only
857 * pre-production machines. */
858 if (IS_HSW_EARLY_SDV(dev_priv))
859 DRM_INFO("This is an early pre-production Haswell machine. "
860 "It may not be fully functional.\n");
861
862 return 0;
863
864err_workqueues:
865 i915_workqueues_cleanup(dev_priv);
866 return ret;
867}
868
869/**
870 * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early()
871 * @dev_priv: device private
872 */
873static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
874{
91c8a326 875 i915_gem_load_cleanup(&dev_priv->drm);
0673ad47
CW		 876 i915_workqueues_cleanup(dev_priv);
877}
878
879static int i915_mmio_setup(struct drm_device *dev)
880{
881 struct drm_i915_private *dev_priv = to_i915(dev);
882 int mmio_bar;
883 int mmio_size;
884
885 mmio_bar = IS_GEN2(dev) ? 1 : 0;
886 /*
887 * Before gen4, the registers and the GTT are behind different BARs.
888 * However, from gen4 onwards, the registers and the GTT are shared
889 * in the same BAR, so we want to restrict this ioremap from
890 * clobbering the GTT which we want ioremap_wc instead. Fortunately,
891 * the register BAR remains the same size for all the earlier
892 * generations up to Ironlake.
893 */
894 if (INTEL_INFO(dev)->gen < 5)
895 mmio_size = 512 * 1024;
896 else
897 mmio_size = 2 * 1024 * 1024;
898 dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, mmio_size);
899 if (dev_priv->regs == NULL) {
900 DRM_ERROR("failed to map registers\n");
901
902 return -EIO;
903 }
904
905 /* Try to make sure MCHBAR is enabled before poking at it */
906 intel_setup_mchbar(dev);
907
908 return 0;
909}
910
911static void i915_mmio_cleanup(struct drm_device *dev)
912{
913 struct drm_i915_private *dev_priv = to_i915(dev);
914
915 intel_teardown_mchbar(dev);
916 pci_iounmap(dev->pdev, dev_priv->regs);
917}
918
919/**
920 * i915_driver_init_mmio - setup device MMIO
921 * @dev_priv: device private
922 *
923 * Setup minimal device state necessary for MMIO accesses later in the
924 * initialization sequence. The setup here should avoid any other device-wide
925 * side effects or exposing the driver via kernel internal or user space
926 * interfaces.
927 */
928static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
929{
91c8a326 930 struct drm_device *dev = &dev_priv->drm;
0673ad47
CW		 931 int ret;
932
933 if (i915_inject_load_failure())
934 return -ENODEV;
935
936 if (i915_get_bridge_dev(dev))
937 return -EIO;
938
939 ret = i915_mmio_setup(dev);
940 if (ret < 0)
941 goto put_bridge;
942
943 intel_uncore_init(dev_priv);
944
945 return 0;
946
947put_bridge:
948 pci_dev_put(dev_priv->bridge_dev);
949
950 return ret;
951}
952
953/**
954 * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio()
955 * @dev_priv: device private
956 */
957static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv)
958{
91c8a326 959 struct drm_device *dev = &dev_priv->drm;
0673ad47
CW		 960
961 intel_uncore_fini(dev_priv);
962 i915_mmio_cleanup(dev);
963 pci_dev_put(dev_priv->bridge_dev);
964}
965
94b4f3ba
CW		 966static void intel_sanitize_options(struct drm_i915_private *dev_priv)
967{
968 i915.enable_execlists =
969 intel_sanitize_enable_execlists(dev_priv,
970 i915.enable_execlists);
971
972 /*
973 * i915.enable_ppgtt is read-only, so do an early pass to validate the
974 * user's requested state against the hardware/driver capabilities. We
975 * do this now so that we can print out any log messages once rather
976 * than every time we check intel_enable_ppgtt().
977 */
978 i915.enable_ppgtt =
979 intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt);
980 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
39df9190
CW		 981
982 i915.semaphores = intel_sanitize_semaphores(dev_priv, i915.semaphores);
983 DRM_DEBUG_DRIVER("use GPU sempahores? %s\n", yesno(i915.semaphores));
94b4f3ba
CW		 984}
985
0673ad47
CW		 986/**
987 * i915_driver_init_hw - setup state requiring device access
988 * @dev_priv: device private
989 *
990 * Setup state that requires accessing the device, but doesn't require
991 * exposing the driver via kernel internal or userspace interfaces.
992 */
993static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
994{
91c8a326 995 struct drm_device *dev = &dev_priv->drm;
0673ad47
CW		 996 struct i915_ggtt *ggtt = &dev_priv->ggtt;
997 uint32_t aperture_size;
998 int ret;
999
1000 if (i915_inject_load_failure())
1001 return -ENODEV;
1002
94b4f3ba
CW		 1003 intel_device_info_runtime_init(dev_priv);
1004
1005 intel_sanitize_options(dev_priv);
0673ad47 1006
0088e522 1007 ret = i915_ggtt_probe_hw(dev);
0673ad47
CW		 1008 if (ret)
1009 return ret;
1010
0673ad47
CW		 1011 /* WARNING: Apparently we must kick fbdev drivers before vgacon,
1012 * otherwise the vga fbdev driver falls over. */
1013 ret = i915_kick_out_firmware_fb(dev_priv);
1014 if (ret) {
1015 DRM_ERROR("failed to remove conflicting framebuffer drivers\n");
1016 goto out_ggtt;
1017 }
1018
1019 ret = i915_kick_out_vgacon(dev_priv);
1020 if (ret) {
1021 DRM_ERROR("failed to remove conflicting VGA console\n");
1022 goto out_ggtt;
1023 }
1024
0088e522
CW		 1025 ret = i915_ggtt_init_hw(dev);
1026 if (ret)
1027 return ret;
1028
1029 ret = i915_ggtt_enable_hw(dev);
1030 if (ret) {
1031 DRM_ERROR("failed to enable GGTT\n");
1032 goto out_ggtt;
1033 }
1034
0673ad47
CW		 1035 pci_set_master(dev->pdev);
1036
1037 /* overlay on gen2 is broken and can't address above 1G */
1038 if (IS_GEN2(dev)) {
1039 ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
1040 if (ret) {
1041 DRM_ERROR("failed to set DMA mask\n");
1042
1043 goto out_ggtt;
1044 }
1045 }
1046
1047
1048 /* 965GM sometimes incorrectly writes to hardware status page (HWS)
1049 * using 32bit addressing, overwriting memory if HWS is located
1050 * above 4GB.
1051 *
1052 * The documentation also mentions an issue with undefined
1053 * behaviour if any general state is accessed within a page above 4GB,
1054 * which also needs to be handled carefully.
1055 */
1056 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) {
1057 ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32));
1058
1059 if (ret) {
1060 DRM_ERROR("failed to set DMA mask\n");
1061
1062 goto out_ggtt;
1063 }
1064 }
1065
1066 aperture_size = ggtt->mappable_end;
1067
1068 ggtt->mappable =
1069 io_mapping_create_wc(ggtt->mappable_base,
1070 aperture_size);
1071 if (!ggtt->mappable) {
1072 ret = -EIO;
1073 goto out_ggtt;
1074 }
1075
1076 ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base,
1077 aperture_size);
1078
1079 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
1080 PM_QOS_DEFAULT_VALUE);
1081
1082 intel_uncore_sanitize(dev_priv);
1083
1084 intel_opregion_setup(dev_priv);
1085
1086 i915_gem_load_init_fences(dev_priv);
1087
1088 /* On the 945G/GM, the chipset reports the MSI capability on the
1089 * integrated graphics even though the support isn't actually there
1090 * according to the published specs. It doesn't appear to function
1091 * correctly in testing on 945G.
1092 * This may be a side effect of MSI having been made available for PEG
1093 * and the registers being closely associated.
1094 *
1095 * According to chipset errata, on the 965GM, MSI interrupts may
1096 * be lost or delayed, but we use them anyways to avoid
1097 * stuck interrupts on some machines.
1098 */
1099 if (!IS_I945G(dev) && !IS_I945GM(dev)) {
1100 if (pci_enable_msi(dev->pdev) < 0)
1101 DRM_DEBUG_DRIVER("can't enable MSI");
1102 }
1103
1104 return 0;
1105
1106out_ggtt:
1107 i915_ggtt_cleanup_hw(dev);
1108
1109 return ret;
1110}
1111
1112/**
1113 * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw()
1114 * @dev_priv: device private
1115 */
1116static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv)
1117{
91c8a326 1118 struct drm_device *dev = &dev_priv->drm;
0673ad47
CW		 1119 struct i915_ggtt *ggtt = &dev_priv->ggtt;
1120
1121 if (dev->pdev->msi_enabled)
1122 pci_disable_msi(dev->pdev);
1123
1124 pm_qos_remove_request(&dev_priv->pm_qos);
1125 arch_phys_wc_del(ggtt->mtrr);
1126 io_mapping_free(ggtt->mappable);
1127 i915_ggtt_cleanup_hw(dev);
1128}
1129
1130/**
1131 * i915_driver_register - register the driver with the rest of the system
1132 * @dev_priv: device private
1133 *
1134 * Perform any steps necessary to make the driver available via kernel
1135 * internal or userspace interfaces.
1136 */
1137static void i915_driver_register(struct drm_i915_private *dev_priv)
1138{
91c8a326 1139 struct drm_device *dev = &dev_priv->drm;
0673ad47
CW		 1140
1141 i915_gem_shrinker_init(dev_priv);
1142
1143 /*
1144 * Notify a valid surface after modesetting,
1145 * when running inside a VM.
1146 */
1147 if (intel_vgpu_active(dev_priv))
1148 I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY);
1149
1150 /* Reveal our presence to userspace */
1151 if (drm_dev_register(dev, 0) == 0) {
1152 i915_debugfs_register(dev_priv);
1153 i915_setup_sysfs(dev);
1154 } else
1155 DRM_ERROR("Failed to register driver for userspace access!\n");
1156
1157 if (INTEL_INFO(dev_priv)->num_pipes) {
1158 /* Must be done after probing outputs */
1159 intel_opregion_register(dev_priv);
1160 acpi_video_register();
1161 }
1162
1163 if (IS_GEN5(dev_priv))
1164 intel_gpu_ips_init(dev_priv);
1165
1166 i915_audio_component_init(dev_priv);
1167
1168 /*
1169 * Some ports require correctly set-up hpd registers for detection to
1170 * work properly (leading to ghost connected connector status), e.g. VGA
1171 * on gm45. Hence we can only set up the initial fbdev config after hpd
1172 * irqs are fully enabled. We do it last so that the async config
1173 * cannot run before the connectors are registered.
1174 */
1175 intel_fbdev_initial_config_async(dev);
1176}
1177
1178/**
1179 * i915_driver_unregister - cleanup the registration done in i915_driver_regiser()
1180 * @dev_priv: device private
1181 */
1182static void i915_driver_unregister(struct drm_i915_private *dev_priv)
1183{
1184 i915_audio_component_cleanup(dev_priv);
1185
1186 intel_gpu_ips_teardown();
1187 acpi_video_unregister();
1188 intel_opregion_unregister(dev_priv);
1189
91c8a326 1190 i915_teardown_sysfs(&dev_priv->drm);
0673ad47 1191 i915_debugfs_unregister(dev_priv);
91c8a326 1192 drm_dev_unregister(&dev_priv->drm);
0673ad47
CW		 1193
1194 i915_gem_shrinker_cleanup(dev_priv);
1195}
1196
1197/**
1198 * i915_driver_load - setup chip and create an initial config
1199 * @dev: DRM device
1200 * @flags: startup flags
1201 *
1202 * The driver load routine has to do several things:
1203 * - drive output discovery via intel_modeset_init()
1204 * - initialize the memory manager
1205 * - allocate initial config memory
1206 * - setup the DRM framebuffer with the allocated memory
1207 */
42f5551d 1208int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
0673ad47
CW		 1209{
1210 struct drm_i915_private *dev_priv;
1211 int ret;
7d87a7f7 1212
a09d0ba1
CW		 1213 if (i915.nuclear_pageflip)
1214 driver.driver_features |= DRIVER_ATOMIC;
1215
0673ad47
CW		 1216 ret = -ENOMEM;
1217 dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
1218 if (dev_priv)
1219 ret = drm_dev_init(&dev_priv->drm, &driver, &pdev->dev);
1220 if (ret) {
1221 dev_printk(KERN_ERR, &pdev->dev,
1222 "[" DRM_NAME ":%s] allocation failed\n", __func__);
1223 kfree(dev_priv);
1224 return ret;
1225 }
72bbf0af 1226
0673ad47
CW		 1227 dev_priv->drm.pdev = pdev;
1228 dev_priv->drm.dev_private = dev_priv;
719388e1 1229
0673ad47
CW		 1230 ret = pci_enable_device(pdev);
1231 if (ret)
1232 goto out_free_priv;
1347f5b4 1233
0673ad47 1234 pci_set_drvdata(pdev, &dev_priv->drm);
ef11bdb3 1235
0673ad47
CW		 1236 ret = i915_driver_init_early(dev_priv, ent);
1237 if (ret < 0)
1238 goto out_pci_disable;
ef11bdb3 1239
0673ad47 1240 intel_runtime_pm_get(dev_priv);
1da177e4 1241
0673ad47
CW		 1242 ret = i915_driver_init_mmio(dev_priv);
1243 if (ret < 0)
1244 goto out_runtime_pm_put;
79e53945 1245
0673ad47
CW		 1246 ret = i915_driver_init_hw(dev_priv);
1247 if (ret < 0)
1248 goto out_cleanup_mmio;
30c964a6
RB		 1249
1250 /*
0673ad47
CW		 1251 * TODO: move the vblank init and parts of modeset init steps into one
1252 * of the i915_driver_init_/i915_driver_register functions according
1253 * to the role/effect of the given init step.
30c964a6 1254 */
0673ad47 1255 if (INTEL_INFO(dev_priv)->num_pipes) {
91c8a326 1256 ret = drm_vblank_init(&dev_priv->drm,
0673ad47
CW		 1257 INTEL_INFO(dev_priv)->num_pipes);
1258 if (ret)
1259 goto out_cleanup_hw;
30c964a6
RB		 1260 }
1261
91c8a326 1262 ret = i915_load_modeset_init(&dev_priv->drm);
0673ad47
CW		 1263 if (ret < 0)
1264 goto out_cleanup_vblank;
1265
1266 i915_driver_register(dev_priv);
1267
1268 intel_runtime_pm_enable(dev_priv);
1269
1270 intel_runtime_pm_put(dev_priv);
1271
1272 return 0;
1273
1274out_cleanup_vblank:
91c8a326 1275 drm_vblank_cleanup(&dev_priv->drm);
0673ad47
CW		 1276out_cleanup_hw:
1277 i915_driver_cleanup_hw(dev_priv);
1278out_cleanup_mmio:
1279 i915_driver_cleanup_mmio(dev_priv);
1280out_runtime_pm_put:
1281 intel_runtime_pm_put(dev_priv);
1282 i915_driver_cleanup_early(dev_priv);
1283out_pci_disable:
1284 pci_disable_device(pdev);
1285out_free_priv:
1286 i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret);
1287 drm_dev_unref(&dev_priv->drm);
30c964a6
RB		 1288 return ret;
1289}
1290
42f5551d 1291void i915_driver_unload(struct drm_device *dev)
3bad0781 1292{
fac5e23e 1293 struct drm_i915_private *dev_priv = to_i915(dev);
3bad0781 1294
0673ad47
CW		 1295 intel_fbdev_fini(dev);
1296
42f5551d
CW		 1297 if (i915_gem_suspend(dev))
1298 DRM_ERROR("failed to idle hardware; continuing to unload!\n");
ce1bb329 1299
0673ad47
CW		 1300 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
1301
1302 i915_driver_unregister(dev_priv);
1303
1304 drm_vblank_cleanup(dev);
1305
1306 intel_modeset_cleanup(dev);
1307
3bad0781 1308 /*
0673ad47
CW		 1309 * free the memory space allocated for the child device
1310 * config parsed from VBT
3bad0781 1311 */
0673ad47
CW		 1312 if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) {
1313 kfree(dev_priv->vbt.child_dev);
1314 dev_priv->vbt.child_dev = NULL;
1315 dev_priv->vbt.child_dev_num = 0;
1316 }
1317 kfree(dev_priv->vbt.sdvo_lvds_vbt_mode);
1318 dev_priv->vbt.sdvo_lvds_vbt_mode = NULL;
1319 kfree(dev_priv->vbt.lfp_lvds_vbt_mode);
1320 dev_priv->vbt.lfp_lvds_vbt_mode = NULL;
3bad0781 1321
0673ad47
CW		 1322 vga_switcheroo_unregister_client(dev->pdev);
1323 vga_client_register(dev->pdev, NULL, NULL, NULL);
bcdb72ac 1324
0673ad47 1325 intel_csr_ucode_fini(dev_priv);
bcdb72ac 1326
0673ad47
CW		 1327 /* Free error state after interrupts are fully disabled. */
1328 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
1329 i915_destroy_error_state(dev);
1330
1331 /* Flush any outstanding unpin_work. */
b7137e0c 1332 drain_workqueue(dev_priv->wq);
0673ad47
CW		 1333
1334 intel_guc_fini(dev);
1335 i915_gem_fini(dev);
1336 intel_fbc_cleanup_cfb(dev_priv);
1337
1338 intel_power_domains_fini(dev_priv);
1339
1340 i915_driver_cleanup_hw(dev_priv);
1341 i915_driver_cleanup_mmio(dev_priv);
1342
1343 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
1344
1345 i915_driver_cleanup_early(dev_priv);
3bad0781
ZW		 1346}
1347
0673ad47 1348static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
2911a35b 1349{
0673ad47 1350 int ret;
2911a35b 1351
0673ad47
CW		 1352 ret = i915_gem_open(dev, file);
1353 if (ret)
1354 return ret;
2911a35b 1355
0673ad47
CW		 1356 return 0;
1357}
71386ef9 1358
0673ad47
CW		 1359/**
1360 * i915_driver_lastclose - clean up after all DRM clients have exited
1361 * @dev: DRM device
1362 *
1363 * Take care of cleaning up after all DRM clients have exited. In the
1364 * mode setting case, we want to restore the kernel's initial mode (just
1365 * in case the last client left us in a bad state).
1366 *
1367 * Additionally, in the non-mode setting case, we'll tear down the GTT
1368 * and DMA structures, since the kernel won't be using them, and clea
1369 * up any GEM state.
1370 */
1371static void i915_driver_lastclose(struct drm_device *dev)
1372{
1373 intel_fbdev_restore_mode(dev);
1374 vga_switcheroo_process_delayed_switch();
1375}
2911a35b 1376
0673ad47
CW		 1377static void i915_driver_preclose(struct drm_device *dev, struct drm_file *file)
1378{
1379 mutex_lock(&dev->struct_mutex);
1380 i915_gem_context_close(dev, file);
1381 i915_gem_release(dev, file);
1382 mutex_unlock(&dev->struct_mutex);
1383}
1384
1385static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
1386{
1387 struct drm_i915_file_private *file_priv = file->driver_priv;
1388
1389 kfree(file_priv);
2911a35b
BW		 1390}
1391
07f9cd0b
ID		 1392static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
1393{
91c8a326 1394 struct drm_device *dev = &dev_priv->drm;
19c8054c 1395 struct intel_encoder *encoder;
07f9cd0b
ID		 1396
1397 drm_modeset_lock_all(dev);
19c8054c
JN		 1398 for_each_intel_encoder(dev, encoder)
1399 if (encoder->suspend)
1400 encoder->suspend(encoder);
07f9cd0b
ID		 1401 drm_modeset_unlock_all(dev);
1402}
1403
1a5df187
PZ		 1404static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
1405 bool rpm_resume);
507e126e 1406static int vlv_suspend_complete(struct drm_i915_private *dev_priv);
f75a1985 1407
bc87229f
ID		 1408static bool suspend_to_idle(struct drm_i915_private *dev_priv)
1409{
1410#if IS_ENABLED(CONFIG_ACPI_SLEEP)
1411 if (acpi_target_system_state() < ACPI_STATE_S3)
1412 return true;
1413#endif
1414 return false;
1415}
ebc32824 1416
5e365c39 1417static int i915_drm_suspend(struct drm_device *dev)
ba8bbcf6 1418{
fac5e23e 1419 struct drm_i915_private *dev_priv = to_i915(dev);
e5747e3a 1420 pci_power_t opregion_target_state;
d5818938 1421 int error;
61caf87c 1422
b8efb17b
ZR		 1423 /* ignore lid events during suspend */
1424 mutex_lock(&dev_priv->modeset_restore_lock);
1425 dev_priv->modeset_restore = MODESET_SUSPENDED;
1426 mutex_unlock(&dev_priv->modeset_restore_lock);
1427
1f814dac
ID		 1428 disable_rpm_wakeref_asserts(dev_priv);
1429
c67a470b
PZ		 1430 /* We do a lot of poking in a lot of registers, make sure they work
1431 * properly. */
da7e29bd 1432 intel_display_set_init_power(dev_priv, true);
cb10799c 1433
5bcf719b
DA		 1434 drm_kms_helper_poll_disable(dev);
1435
ba8bbcf6 1436 pci_save_state(dev->pdev);
ba8bbcf6 1437
d5818938
DV		 1438 error = i915_gem_suspend(dev);
1439 if (error) {
1440 dev_err(&dev->pdev->dev,
1441 "GEM idle failed, resume might fail\n");
1f814dac 1442 goto out;
d5818938 1443 }
db1b76ca 1444
a1c41994
AD		 1445 intel_guc_suspend(dev);
1446
6b72d486 1447 intel_display_suspend(dev);
2eb5252e 1448
d5818938 1449 intel_dp_mst_suspend(dev);
7d708ee4 1450
d5818938
DV		 1451 intel_runtime_pm_disable_interrupts(dev_priv);
1452 intel_hpd_cancel_work(dev_priv);
09b64267 1453
d5818938 1454 intel_suspend_encoders(dev_priv);
0e32b39c 1455
d5818938 1456 intel_suspend_hw(dev);
5669fcac 1457
828c7908
BW		 1458 i915_gem_suspend_gtt_mappings(dev);
1459
9e06dd39
JB		 1460 i915_save_state(dev);
1461
bc87229f 1462 opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
6f9f4b7a 1463 intel_opregion_notify_adapter(dev_priv, opregion_target_state);
e5747e3a 1464
dc97997a 1465 intel_uncore_forcewake_reset(dev_priv, false);
03d92e47 1466 intel_opregion_unregister(dev_priv);
8ee1c3db 1467
82e3b8c1 1468 intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
3fa016a0 1469
62d5d69b
MK		 1470 dev_priv->suspend_count++;
1471
85e90679
KCA		 1472 intel_display_set_init_power(dev_priv, false);
1473
f74ed08d 1474 intel_csr_ucode_suspend(dev_priv);
f514c2d8 1475
1f814dac
ID		 1476out:
1477 enable_rpm_wakeref_asserts(dev_priv);
1478
1479 return error;
84b79f8d
RW		 1480}
1481
ab3be73f 1482static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation)
c3c09c95 1483{
fac5e23e 1484 struct drm_i915_private *dev_priv = to_i915(drm_dev);
bc87229f 1485 bool fw_csr;
c3c09c95
ID		 1486 int ret;
1487
1f814dac
ID		 1488 disable_rpm_wakeref_asserts(dev_priv);
1489
a7c8125f
ID		 1490 fw_csr = !IS_BROXTON(dev_priv) &&
1491 suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload;
bc87229f
ID		 1492 /*
1493 * In case of firmware assisted context save/restore don't manually
1494 * deinit the power domains. This also means the CSR/DMC firmware will
1495 * stay active, it will power down any HW resources as required and
1496 * also enable deeper system power states that would be blocked if the
1497 * firmware was inactive.
1498 */
1499 if (!fw_csr)
1500 intel_power_domains_suspend(dev_priv);
73dfc227 1501
507e126e 1502 ret = 0;
b8aea3d1 1503 if (IS_BROXTON(dev_priv))
507e126e 1504 bxt_enable_dc9(dev_priv);
b8aea3d1 1505 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
507e126e
ID		 1506 hsw_enable_pc8(dev_priv);
1507 else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1508 ret = vlv_suspend_complete(dev_priv);
c3c09c95
ID		 1509
1510 if (ret) {
1511 DRM_ERROR("Suspend complete failed: %d\n", ret);
bc87229f
ID		 1512 if (!fw_csr)
1513 intel_power_domains_init_hw(dev_priv, true);
c3c09c95 1514
1f814dac 1515 goto out;
c3c09c95
ID		 1516 }
1517
1518 pci_disable_device(drm_dev->pdev);
ab3be73f 1519 /*
54875571 1520 * During hibernation on some platforms the BIOS may try to access
ab3be73f
ID		 1521 * the device even though it's already in D3 and hang the machine. So
1522 * leave the device in D0 on those platforms and hope the BIOS will
54875571
ID		 1523 * power down the device properly. The issue was seen on multiple old
1524 * GENs with different BIOS vendors, so having an explicit blacklist
1525 * is inpractical; apply the workaround on everything pre GEN6. The
1526 * platforms where the issue was seen:
1527 * Lenovo Thinkpad X301, X61s, X60, T60, X41
1528 * Fujitsu FSC S7110
1529 * Acer Aspire 1830T
ab3be73f 1530 */
54875571 1531 if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6))
ab3be73f 1532 pci_set_power_state(drm_dev->pdev, PCI_D3hot);
c3c09c95 1533
bc87229f
ID		 1534 dev_priv->suspended_to_idle = suspend_to_idle(dev_priv);
1535
1f814dac
ID		 1536out:
1537 enable_rpm_wakeref_asserts(dev_priv);
1538
1539 return ret;
c3c09c95
ID		 1540}
1541
1751fcf9 1542int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state)
84b79f8d
RW		 1543{
1544 int error;
1545
ded8b07d 1546 if (!dev) {
84b79f8d
RW		 1547 DRM_ERROR("dev: %p\n", dev);
1548 DRM_ERROR("DRM not initialized, aborting suspend.\n");
1549 return -ENODEV;
1550 }
1551
0b14cbd2
ID		 1552 if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND &&
1553 state.event != PM_EVENT_FREEZE))
1554 return -EINVAL;
5bcf719b
DA		 1555
1556 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1557 return 0;
6eecba33 1558
5e365c39 1559 error = i915_drm_suspend(dev);
84b79f8d
RW		 1560 if (error)
1561 return error;
1562
ab3be73f 1563 return i915_drm_suspend_late(dev, false);
ba8bbcf6
JB		 1564}
1565
5e365c39 1566static int i915_drm_resume(struct drm_device *dev)
76c4b250 1567{
fac5e23e 1568 struct drm_i915_private *dev_priv = to_i915(dev);
ac840ae5 1569 int ret;
9d49c0ef 1570
1f814dac
ID		 1571 disable_rpm_wakeref_asserts(dev_priv);
1572
ac840ae5
VS		 1573 ret = i915_ggtt_enable_hw(dev);
1574 if (ret)
1575 DRM_ERROR("failed to re-enable GGTT\n");
1576
f74ed08d
ID		 1577 intel_csr_ucode_resume(dev_priv);
1578
5ab57c70 1579 i915_gem_resume(dev);
9d49c0ef 1580
61caf87c 1581 i915_restore_state(dev);
6f9f4b7a 1582 intel_opregion_setup(dev_priv);
61caf87c 1583
d5818938
DV		 1584 intel_init_pch_refclk(dev);
1585 drm_mode_config_reset(dev);
1833b134 1586
364aece0
PA		 1587 /*
1588 * Interrupts have to be enabled before any batches are run. If not the
1589 * GPU will hang. i915_gem_init_hw() will initiate batches to
1590 * update/restore the context.
1591 *
1592 * Modeset enabling in intel_modeset_init_hw() also needs working
1593 * interrupts.
1594 */
1595 intel_runtime_pm_enable_interrupts(dev_priv);
1596
d5818938
DV		 1597 mutex_lock(&dev->struct_mutex);
1598 if (i915_gem_init_hw(dev)) {
1599 DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
338d0eea 1600 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
d5818938
DV		 1601 }
1602 mutex_unlock(&dev->struct_mutex);
226485e9 1603
a1c41994
AD		 1604 intel_guc_resume(dev);
1605
d5818938 1606 intel_modeset_init_hw(dev);
24576d23 1607
d5818938
DV		 1608 spin_lock_irq(&dev_priv->irq_lock);
1609 if (dev_priv->display.hpd_irq_setup)
91d14251 1610 dev_priv->display.hpd_irq_setup(dev_priv);
d5818938 1611 spin_unlock_irq(&dev_priv->irq_lock);
0e32b39c 1612
d5818938 1613 intel_dp_mst_resume(dev);
e7d6f7d7 1614
a16b7658
L		 1615 intel_display_resume(dev);
1616
d5818938
DV		 1617 /*
1618 * ... but also need to make sure that hotplug processing
1619 * doesn't cause havoc. Like in the driver load code we don't
1620 * bother with the tiny race here where we might loose hotplug
1621 * notifications.
1622 * */
1623 intel_hpd_init(dev_priv);
1624 /* Config may have changed between suspend and resume */
1625 drm_helper_hpd_irq_event(dev);
1daed3fb 1626
03d92e47 1627 intel_opregion_register(dev_priv);
44834a67 1628
82e3b8c1 1629 intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
073f34d9 1630
b8efb17b
ZR		 1631 mutex_lock(&dev_priv->modeset_restore_lock);
1632 dev_priv->modeset_restore = MODESET_DONE;
1633 mutex_unlock(&dev_priv->modeset_restore_lock);
8a187455 1634
6f9f4b7a 1635 intel_opregion_notify_adapter(dev_priv, PCI_D0);
e5747e3a 1636
54b4f68f 1637 intel_autoenable_gt_powersave(dev_priv);
ee6f280e
ID		 1638 drm_kms_helper_poll_enable(dev);
1639
1f814dac
ID		 1640 enable_rpm_wakeref_asserts(dev_priv);
1641
074c6ada 1642 return 0;
84b79f8d
RW		 1643}
1644
5e365c39 1645static int i915_drm_resume_early(struct drm_device *dev)
84b79f8d 1646{
fac5e23e 1647 struct drm_i915_private *dev_priv = to_i915(dev);
44410cd0 1648 int ret;
36d61e67 1649
76c4b250
ID		 1650 /*
1651 * We have a resume ordering issue with the snd-hda driver also
1652 * requiring our device to be power up. Due to the lack of a
1653 * parent/child relationship we currently solve this with an early
1654 * resume hook.
1655 *
1656 * FIXME: This should be solved with a special hdmi sink device or
1657 * similar so that power domains can be employed.
1658 */
44410cd0
ID		 1659
1660 /*
1661 * Note that we need to set the power state explicitly, since we
1662 * powered off the device during freeze and the PCI core won't power
1663 * it back up for us during thaw. Powering off the device during
1664 * freeze is not a hard requirement though, and during the
1665 * suspend/resume phases the PCI core makes sure we get here with the
1666 * device powered on. So in case we change our freeze logic and keep
1667 * the device powered we can also remove the following set power state
1668 * call.
1669 */
1670 ret = pci_set_power_state(dev->pdev, PCI_D0);
1671 if (ret) {
1672 DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret);
1673 goto out;
1674 }
1675
1676 /*
1677 * Note that pci_enable_device() first enables any parent bridge
1678 * device and only then sets the power state for this device. The
1679 * bridge enabling is a nop though, since bridge devices are resumed
1680 * first. The order of enabling power and enabling the device is
1681 * imposed by the PCI core as described above, so here we preserve the
1682 * same order for the freeze/thaw phases.
1683 *
1684 * TODO: eventually we should remove pci_disable_device() /
1685 * pci_enable_enable_device() from suspend/resume. Due to how they
1686 * depend on the device enable refcount we can't anyway depend on them
1687 * disabling/enabling the device.
1688 */
bc87229f
ID		 1689 if (pci_enable_device(dev->pdev)) {
1690 ret = -EIO;
1691 goto out;
1692 }
84b79f8d
RW		 1693
1694 pci_set_master(dev->pdev);
1695
1f814dac
ID		 1696 disable_rpm_wakeref_asserts(dev_priv);
1697
666a4537 1698 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1a5df187 1699 ret = vlv_resume_prepare(dev_priv, false);
36d61e67 1700 if (ret)
ff0b187f
DL		 1701 DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
1702 ret);
36d61e67 1703
dc97997a 1704 intel_uncore_early_sanitize(dev_priv, true);
efee833a 1705
dc97997a 1706 if (IS_BROXTON(dev_priv)) {
da2f41d1
ID		 1707 if (!dev_priv->suspended_to_idle)
1708 gen9_sanitize_dc_state(dev_priv);
507e126e 1709 bxt_disable_dc9(dev_priv);
da2f41d1 1710 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
a9a6b73a 1711 hsw_disable_pc8(dev_priv);
da2f41d1 1712 }
efee833a 1713
dc97997a 1714 intel_uncore_sanitize(dev_priv);
bc87229f 1715
a7c8125f
ID		 1716 if (IS_BROXTON(dev_priv) ||
1717 !(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload))
bc87229f
ID		 1718 intel_power_domains_init_hw(dev_priv, true);
1719
6e35e8ab
ID		 1720 enable_rpm_wakeref_asserts(dev_priv);
1721
bc87229f
ID		 1722out:
1723 dev_priv->suspended_to_idle = false;
36d61e67
ID		 1724
1725 return ret;
76c4b250
ID		 1726}
1727
1751fcf9 1728int i915_resume_switcheroo(struct drm_device *dev)
76c4b250 1729{
50a0072f 1730 int ret;
76c4b250 1731
097dd837
ID		 1732 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1733 return 0;
1734
5e365c39 1735 ret = i915_drm_resume_early(dev);
50a0072f
ID		 1736 if (ret)
1737 return ret;
1738
5a17514e
ID		 1739 return i915_drm_resume(dev);
1740}
1741
11ed50ec 1742/**
f3953dcb 1743 * i915_reset - reset chip after a hang
11ed50ec 1744 * @dev: drm device to reset
11ed50ec
BG		 1745 *
1746 * Reset the chip. Useful if a hang is detected. Returns zero on successful
1747 * reset or otherwise an error code.
1748 *
1749 * Procedure is fairly simple:
1750 * - reset the chip using the reset reg
1751 * - re-init context state
1752 * - re-init hardware status page
1753 * - re-init ring buffer
1754 * - re-init interrupt state
1755 * - re-init display
1756 */
c033666a 1757int i915_reset(struct drm_i915_private *dev_priv)
11ed50ec 1758{
91c8a326 1759 struct drm_device *dev = &dev_priv->drm;
d98c52cf
CW		 1760 struct i915_gpu_error *error = &dev_priv->gpu_error;
1761 unsigned reset_counter;
0573ed4a 1762 int ret;
11ed50ec 1763
d54a02c0 1764 mutex_lock(&dev->struct_mutex);
11ed50ec 1765
d98c52cf
CW		 1766 /* Clear any previous failed attempts at recovery. Time to try again. */
1767 atomic_andnot(I915_WEDGED, &error->reset_counter);
77f01230 1768
d98c52cf
CW		 1769 /* Clear the reset-in-progress flag and increment the reset epoch. */
1770 reset_counter = atomic_inc_return(&error->reset_counter);
1771 if (WARN_ON(__i915_reset_in_progress(reset_counter))) {
1772 ret = -EIO;
1773 goto error;
1774 }
1775
7b4d3a16
CW		 1776 pr_notice("drm/i915: Resetting chip after gpu hang\n");
1777
d98c52cf 1778 i915_gem_reset(dev);
2e7c8ee7 1779
dc97997a 1780 ret = intel_gpu_reset(dev_priv, ALL_ENGINES);
0573ed4a 1781 if (ret) {
804e59a8
CW		 1782 if (ret != -ENODEV)
1783 DRM_ERROR("Failed to reset chip: %i\n", ret);
1784 else
1785 DRM_DEBUG_DRIVER("GPU reset disabled\n");
d98c52cf 1786 goto error;
11ed50ec
BG		 1787 }
1788
1362b776
VS		 1789 intel_overlay_reset(dev_priv);
1790
11ed50ec
BG		 1791 /* Ok, now get things going again... */
1792
1793 /*
1794 * Everything depends on having the GTT running, so we need to start
1795 * there. Fortunately we don't need to do this unless we reset the
1796 * chip at a PCI level.
1797 *
1798 * Next we need to restore the context, but we don't use those
1799 * yet either...
1800 *
1801 * Ring buffer needs to be re-initialized in the KMS case, or if X
1802 * was running at the time of the reset (i.e. we weren't VT
1803 * switched away).
1804 */
33d30a9c 1805 ret = i915_gem_init_hw(dev);
33d30a9c
DV		 1806 if (ret) {
1807 DRM_ERROR("Failed hw init on reset %d\n", ret);
d98c52cf 1808 goto error;
11ed50ec
BG		 1809 }
1810
d98c52cf
CW		 1811 mutex_unlock(&dev->struct_mutex);
1812
33d30a9c
DV		 1813 /*
1814 * rps/rc6 re-init is necessary to restore state lost after the
1815 * reset and the re-install of gt irqs. Skip for ironlake per
1816 * previous concerns that it doesn't respond well to some forms
1817 * of re-init after reset.
1818 */
54b4f68f 1819 intel_autoenable_gt_powersave(dev_priv);
33d30a9c 1820
11ed50ec 1821 return 0;
d98c52cf
CW		 1822
1823error:
1824 atomic_or(I915_WEDGED, &error->reset_counter);
1825 mutex_unlock(&dev->struct_mutex);
1826 return ret;
11ed50ec
BG		 1827}
1828
84b79f8d 1829static int i915_pm_suspend(struct device *dev)
112b715e 1830{
84b79f8d
RW		 1831 struct pci_dev *pdev = to_pci_dev(dev);
1832 struct drm_device *drm_dev = pci_get_drvdata(pdev);
112b715e 1833
ded8b07d 1834 if (!drm_dev) {
84b79f8d
RW		 1835 dev_err(dev, "DRM not initialized, aborting suspend.\n");
1836 return -ENODEV;
1837 }
112b715e 1838
5bcf719b
DA		 1839 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1840 return 0;
1841
5e365c39 1842 return i915_drm_suspend(drm_dev);
76c4b250
ID		 1843}
1844
1845static int i915_pm_suspend_late(struct device *dev)
1846{
91c8a326 1847 struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
76c4b250
ID		 1848
1849 /*
c965d995 1850 * We have a suspend ordering issue with the snd-hda driver also
76c4b250
ID		 1851 * requiring our device to be power up. Due to the lack of a
1852 * parent/child relationship we currently solve this with an late
1853 * suspend hook.
1854 *
1855 * FIXME: This should be solved with a special hdmi sink device or
1856 * similar so that power domains can be employed.
1857 */
1858 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1859 return 0;
112b715e 1860
ab3be73f
ID		 1861 return i915_drm_suspend_late(drm_dev, false);
1862}
1863
1864static int i915_pm_poweroff_late(struct device *dev)
1865{
91c8a326 1866 struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
ab3be73f
ID		 1867
1868 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1869 return 0;
1870
1871 return i915_drm_suspend_late(drm_dev, true);
cbda12d7
ZW		 1872}
1873
76c4b250
ID		 1874static int i915_pm_resume_early(struct device *dev)
1875{
91c8a326 1876 struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
76c4b250 1877
097dd837
ID		 1878 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1879 return 0;
1880
5e365c39 1881 return i915_drm_resume_early(drm_dev);
76c4b250
ID		 1882}
1883
84b79f8d 1884static int i915_pm_resume(struct device *dev)
cbda12d7 1885{
91c8a326 1886 struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
84b79f8d 1887
097dd837
ID		 1888 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1889 return 0;
1890
5a17514e 1891 return i915_drm_resume(drm_dev);
cbda12d7
ZW		 1892}
1893
1f19ac2a
CW		 1894/* freeze: before creating the hibernation_image */
1895static int i915_pm_freeze(struct device *dev)
1896{
1897 return i915_pm_suspend(dev);
1898}
1899
1900static int i915_pm_freeze_late(struct device *dev)
1901{
461fb99c
CW		 1902 int ret;
1903
1904 ret = i915_pm_suspend_late(dev);
1905 if (ret)
1906 return ret;
1907
1908 ret = i915_gem_freeze_late(dev_to_i915(dev));
1909 if (ret)
1910 return ret;
1911
1912 return 0;
1f19ac2a
CW		 1913}
1914
1915/* thaw: called after creating the hibernation image, but before turning off. */
1916static int i915_pm_thaw_early(struct device *dev)
1917{
1918 return i915_pm_resume_early(dev);
1919}
1920
1921static int i915_pm_thaw(struct device *dev)
1922{
1923 return i915_pm_resume(dev);
1924}
1925
1926/* restore: called after loading the hibernation image. */
1927static int i915_pm_restore_early(struct device *dev)
1928{
1929 return i915_pm_resume_early(dev);
1930}
1931
1932static int i915_pm_restore(struct device *dev)
1933{
1934 return i915_pm_resume(dev);
1935}
1936
ddeea5b0
ID		 1937/*
1938 * Save all Gunit registers that may be lost after a D3 and a subsequent
1939 * S0i[R123] transition. The list of registers needing a save/restore is
1940 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
1941 * registers in the following way:
1942 * - Driver: saved/restored by the driver
1943 * - Punit : saved/restored by the Punit firmware
1944 * - No, w/o marking: no need to save/restore, since the register is R/O or
1945 * used internally by the HW in a way that doesn't depend
1946 * keeping the content across a suspend/resume.
1947 * - Debug : used for debugging
1948 *
1949 * We save/restore all registers marked with 'Driver', with the following
1950 * exceptions:
1951 * - Registers out of use, including also registers marked with 'Debug'.
1952 * These have no effect on the driver's operation, so we don't save/restore
1953 * them to reduce the overhead.
1954 * - Registers that are fully setup by an initialization function called from
1955 * the resume path. For example many clock gating and RPS/RC6 registers.
1956 * - Registers that provide the right functionality with their reset defaults.
1957 *
1958 * TODO: Except for registers that based on the above 3 criteria can be safely
1959 * ignored, we save/restore all others, practically treating the HW context as
1960 * a black-box for the driver. Further investigation is needed to reduce the
1961 * saved/restored registers even further, by following the same 3 criteria.
1962 */
1963static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv)
1964{
1965 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
1966 int i;
1967
1968 /* GAM 0x4000-0x4770 */
1969 s->wr_watermark = I915_READ(GEN7_WR_WATERMARK);
1970 s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL);
1971 s->arb_mode = I915_READ(ARB_MODE);
1972 s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0);
1973 s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1);
1974
1975 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
22dfe79f 1976 s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
ddeea5b0
ID		 1977
1978 s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
b5f1c97f 1979 s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT);
ddeea5b0
ID		 1980
1981 s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7);
1982 s->ecochk = I915_READ(GAM_ECOCHK);
1983 s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7);
1984 s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7);
1985
1986 s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR);
1987
1988 /* MBC 0x9024-0x91D0, 0x8500 */
1989 s->g3dctl = I915_READ(VLV_G3DCTL);
1990 s->gsckgctl = I915_READ(VLV_GSCKGCTL);
1991 s->mbctl = I915_READ(GEN6_MBCTL);
1992
1993 /* GCP 0x9400-0x9424, 0x8100-0x810C */
1994 s->ucgctl1 = I915_READ(GEN6_UCGCTL1);
1995 s->ucgctl3 = I915_READ(GEN6_UCGCTL3);
1996 s->rcgctl1 = I915_READ(GEN6_RCGCTL1);
1997 s->rcgctl2 = I915_READ(GEN6_RCGCTL2);
1998 s->rstctl = I915_READ(GEN6_RSTCTL);
1999 s->misccpctl = I915_READ(GEN7_MISCCPCTL);
2000
2001 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
2002 s->gfxpause = I915_READ(GEN6_GFXPAUSE);
2003 s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC);
2004 s->rpdeuc = I915_READ(GEN6_RPDEUC);
2005 s->ecobus = I915_READ(ECOBUS);
2006 s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL);
2007 s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT);
2008 s->rp_deucsw = I915_READ(GEN6_RPDEUCSW);
2009 s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR);
2010 s->rcedata = I915_READ(VLV_RCEDATA);
2011 s->spare2gh = I915_READ(VLV_SPAREG2H);
2012
2013 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
2014 s->gt_imr = I915_READ(GTIMR);
2015 s->gt_ier = I915_READ(GTIER);
2016 s->pm_imr = I915_READ(GEN6_PMIMR);
2017 s->pm_ier = I915_READ(GEN6_PMIER);
2018
2019 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
22dfe79f 2020 s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
ddeea5b0
ID		 2021
2022 /* GT SA CZ domain, 0x100000-0x138124 */
2023 s->tilectl = I915_READ(TILECTL);
2024 s->gt_fifoctl = I915_READ(GTFIFOCTL);
2025 s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL);
2026 s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2027 s->pmwgicz = I915_READ(VLV_PMWGICZ);
2028
2029 /* Gunit-Display CZ domain, 0x182028-0x1821CF */
2030 s->gu_ctl0 = I915_READ(VLV_GU_CTL0);
2031 s->gu_ctl1 = I915_READ(VLV_GU_CTL1);
9c25210f 2032 s->pcbr = I915_READ(VLV_PCBR);
ddeea5b0
ID		 2033 s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2);
2034
2035 /*
2036 * Not saving any of:
2037 * DFT, 0x9800-0x9EC0
2038 * SARB, 0xB000-0xB1FC
2039 * GAC, 0x5208-0x524C, 0x14000-0x14C000
2040 * PCI CFG
2041 */
2042}
2043
2044static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv)
2045{
2046 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
2047 u32 val;
2048 int i;
2049
2050 /* GAM 0x4000-0x4770 */
2051 I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark);
2052 I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl);
2053 I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16));
2054 I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0);
2055 I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1);
2056
2057 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
22dfe79f 2058 I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
ddeea5b0
ID		 2059
2060 I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
b5f1c97f 2061 I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
ddeea5b0
ID		 2062
2063 I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp);
2064 I915_WRITE(GAM_ECOCHK, s->ecochk);
2065 I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp);
2066 I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp);
2067
2068 I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr);
2069
2070 /* MBC 0x9024-0x91D0, 0x8500 */
2071 I915_WRITE(VLV_G3DCTL, s->g3dctl);
2072 I915_WRITE(VLV_GSCKGCTL, s->gsckgctl);
2073 I915_WRITE(GEN6_MBCTL, s->mbctl);
2074
2075 /* GCP 0x9400-0x9424, 0x8100-0x810C */
2076 I915_WRITE(GEN6_UCGCTL1, s->ucgctl1);
2077 I915_WRITE(GEN6_UCGCTL3, s->ucgctl3);
2078 I915_WRITE(GEN6_RCGCTL1, s->rcgctl1);
2079 I915_WRITE(GEN6_RCGCTL2, s->rcgctl2);
2080 I915_WRITE(GEN6_RSTCTL, s->rstctl);
2081 I915_WRITE(GEN7_MISCCPCTL, s->misccpctl);
2082
2083 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
2084 I915_WRITE(GEN6_GFXPAUSE, s->gfxpause);
2085 I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc);
2086 I915_WRITE(GEN6_RPDEUC, s->rpdeuc);
2087 I915_WRITE(ECOBUS, s->ecobus);
2088 I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl);
2089 I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout);
2090 I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw);
2091 I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr);
2092 I915_WRITE(VLV_RCEDATA, s->rcedata);
2093 I915_WRITE(VLV_SPAREG2H, s->spare2gh);
2094
2095 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
2096 I915_WRITE(GTIMR, s->gt_imr);
2097 I915_WRITE(GTIER, s->gt_ier);
2098 I915_WRITE(GEN6_PMIMR, s->pm_imr);
2099 I915_WRITE(GEN6_PMIER, s->pm_ier);
2100
2101 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
22dfe79f 2102 I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
ddeea5b0
ID		 2103
2104 /* GT SA CZ domain, 0x100000-0x138124 */
2105 I915_WRITE(TILECTL, s->tilectl);
2106 I915_WRITE(GTFIFOCTL, s->gt_fifoctl);
2107 /*
2108 * Preserve the GT allow wake and GFX force clock bit, they are not
2109 * be restored, as they are used to control the s0ix suspend/resume
2110 * sequence by the caller.
2111 */
2112 val = I915_READ(VLV_GTLC_WAKE_CTRL);
2113 val &= VLV_GTLC_ALLOWWAKEREQ;
2114 val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
2115 I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2116
2117 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2118 val &= VLV_GFX_CLK_FORCE_ON_BIT;
2119 val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
2120 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2121
2122 I915_WRITE(VLV_PMWGICZ, s->pmwgicz);
2123
2124 /* Gunit-Display CZ domain, 0x182028-0x1821CF */
2125 I915_WRITE(VLV_GU_CTL0, s->gu_ctl0);
2126 I915_WRITE(VLV_GU_CTL1, s->gu_ctl1);
9c25210f 2127 I915_WRITE(VLV_PCBR, s->pcbr);
ddeea5b0
ID		 2128 I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2);
2129}
2130
650ad970
ID		 2131int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
2132{
2133 u32 val;
2134 int err;
2135
650ad970
ID		 2136 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2137 val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
2138 if (force_on)
2139 val |= VLV_GFX_CLK_FORCE_ON_BIT;
2140 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2141
2142 if (!force_on)
2143 return 0;
2144
c6ddc5f3
CW		 2145 err = intel_wait_for_register(dev_priv,
2146 VLV_GTLC_SURVIVABILITY_REG,
2147 VLV_GFX_CLK_STATUS_BIT,
2148 VLV_GFX_CLK_STATUS_BIT,
2149 20);
650ad970
ID		 2150 if (err)
2151 DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n",
2152 I915_READ(VLV_GTLC_SURVIVABILITY_REG));
2153
2154 return err;
650ad970
ID		 2155}
2156
ddeea5b0
ID		 2157static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
2158{
2159 u32 val;
2160 int err = 0;
2161
2162 val = I915_READ(VLV_GTLC_WAKE_CTRL);
2163 val &= ~VLV_GTLC_ALLOWWAKEREQ;
2164 if (allow)
2165 val |= VLV_GTLC_ALLOWWAKEREQ;
2166 I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2167 POSTING_READ(VLV_GTLC_WAKE_CTRL);
2168
b2736695
CW		 2169 err = intel_wait_for_register(dev_priv,
2170 VLV_GTLC_PW_STATUS,
2171 VLV_GTLC_ALLOWWAKEACK,
2172 allow,
2173 1);
ddeea5b0
ID		 2174 if (err)
2175 DRM_ERROR("timeout disabling GT waking\n");
b2736695 2176
ddeea5b0 2177 return err;
ddeea5b0
ID		 2178}
2179
2180static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
2181 bool wait_for_on)
2182{
2183 u32 mask;
2184 u32 val;
2185 int err;
2186
2187 mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
2188 val = wait_for_on ? mask : 0;
41ce405e 2189 if ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val)
ddeea5b0
ID		 2190 return 0;
2191
2192 DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n",
87ad3212
JN		 2193 onoff(wait_for_on),
2194 I915_READ(VLV_GTLC_PW_STATUS));
ddeea5b0
ID		 2195
2196 /*
2197 * RC6 transitioning can be delayed up to 2 msec (see
2198 * valleyview_enable_rps), use 3 msec for safety.
2199 */
41ce405e
CW		 2200 err = intel_wait_for_register(dev_priv,
2201 VLV_GTLC_PW_STATUS, mask, val,
2202 3);
ddeea5b0
ID		 2203 if (err)
2204 DRM_ERROR("timeout waiting for GT wells to go %s\n",
87ad3212 2205 onoff(wait_for_on));
ddeea5b0
ID		 2206
2207 return err;
ddeea5b0
ID		 2208}
2209
2210static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
2211{
2212 if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
2213 return;
2214
6fa283b0 2215 DRM_DEBUG_DRIVER("GT register access while GT waking disabled\n");
ddeea5b0
ID		 2216 I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
2217}
2218
ebc32824 2219static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
ddeea5b0
ID		 2220{
2221 u32 mask;
2222 int err;
2223
2224 /*
2225 * Bspec defines the following GT well on flags as debug only, so
2226 * don't treat them as hard failures.
2227 */
2228 (void)vlv_wait_for_gt_wells(dev_priv, false);
2229
2230 mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
2231 WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask);
2232
2233 vlv_check_no_gt_access(dev_priv);
2234
2235 err = vlv_force_gfx_clock(dev_priv, true);
2236 if (err)
2237 goto err1;
2238
2239 err = vlv_allow_gt_wake(dev_priv, false);
2240 if (err)
2241 goto err2;
98711167 2242
2d1fe073 2243 if (!IS_CHERRYVIEW(dev_priv))
98711167 2244 vlv_save_gunit_s0ix_state(dev_priv);
ddeea5b0
ID		 2245
2246 err = vlv_force_gfx_clock(dev_priv, false);
2247 if (err)
2248 goto err2;
2249
2250 return 0;
2251
2252err2:
2253 /* For safety always re-enable waking and disable gfx clock forcing */
2254 vlv_allow_gt_wake(dev_priv, true);
2255err1:
2256 vlv_force_gfx_clock(dev_priv, false);
2257
2258 return err;
2259}
2260
016970be
SK		 2261static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
2262 bool rpm_resume)
ddeea5b0 2263{
91c8a326 2264 struct drm_device *dev = &dev_priv->drm;
ddeea5b0
ID		 2265 int err;
2266 int ret;
2267
2268 /*
2269 * If any of the steps fail just try to continue, that's the best we
2270 * can do at this point. Return the first error code (which will also
2271 * leave RPM permanently disabled).
2272 */
2273 ret = vlv_force_gfx_clock(dev_priv, true);
2274
2d1fe073 2275 if (!IS_CHERRYVIEW(dev_priv))
98711167 2276 vlv_restore_gunit_s0ix_state(dev_priv);
ddeea5b0
ID		 2277
2278 err = vlv_allow_gt_wake(dev_priv, true);
2279 if (!ret)
2280 ret = err;
2281
2282 err = vlv_force_gfx_clock(dev_priv, false);
2283 if (!ret)
2284 ret = err;
2285
2286 vlv_check_no_gt_access(dev_priv);
2287
016970be
SK		 2288 if (rpm_resume) {
2289 intel_init_clock_gating(dev);
2290 i915_gem_restore_fences(dev);
2291 }
ddeea5b0
ID		 2292
2293 return ret;
2294}
2295
97bea207 2296static int intel_runtime_suspend(struct device *device)
8a187455
PZ		 2297{
2298 struct pci_dev *pdev = to_pci_dev(device);
2299 struct drm_device *dev = pci_get_drvdata(pdev);
fac5e23e 2300 struct drm_i915_private *dev_priv = to_i915(dev);
0ab9cfeb 2301 int ret;
8a187455 2302
dc97997a 2303 if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6())))
c6df39b5
ID		 2304 return -ENODEV;
2305
604effb7
ID		 2306 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
2307 return -ENODEV;
2308
8a187455
PZ		 2309 DRM_DEBUG_KMS("Suspending device\n");
2310
d6102977
ID		 2311 /*
2312 * We could deadlock here in case another thread holding struct_mutex
2313 * calls RPM suspend concurrently, since the RPM suspend will wait
2314 * first for this RPM suspend to finish. In this case the concurrent
2315 * RPM resume will be followed by its RPM suspend counterpart. Still
2316 * for consistency return -EAGAIN, which will reschedule this suspend.
2317 */
2318 if (!mutex_trylock(&dev->struct_mutex)) {
2319 DRM_DEBUG_KMS("device lock contention, deffering suspend\n");
2320 /*
2321 * Bump the expiration timestamp, otherwise the suspend won't
2322 * be rescheduled.
2323 */
2324 pm_runtime_mark_last_busy(device);
2325
2326 return -EAGAIN;
2327 }
1f814dac
ID		 2328
2329 disable_rpm_wakeref_asserts(dev_priv);
2330
d6102977
ID		 2331 /*
2332 * We are safe here against re-faults, since the fault handler takes
2333 * an RPM reference.
2334 */
2335 i915_gem_release_all_mmaps(dev_priv);
2336 mutex_unlock(&dev->struct_mutex);
2337
a1c41994
AD		 2338 intel_guc_suspend(dev);
2339
2eb5252e 2340 intel_runtime_pm_disable_interrupts(dev_priv);
b5478bcd 2341
507e126e
ID		 2342 ret = 0;
2343 if (IS_BROXTON(dev_priv)) {
2344 bxt_display_core_uninit(dev_priv);
2345 bxt_enable_dc9(dev_priv);
2346 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2347 hsw_enable_pc8(dev_priv);
2348 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
2349 ret = vlv_suspend_complete(dev_priv);
2350 }
2351
0ab9cfeb
ID		 2352 if (ret) {
2353 DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
b963291c 2354 intel_runtime_pm_enable_interrupts(dev_priv);
0ab9cfeb 2355
1f814dac
ID		 2356 enable_rpm_wakeref_asserts(dev_priv);
2357
0ab9cfeb
ID		 2358 return ret;
2359 }
a8a8bd54 2360
dc97997a 2361 intel_uncore_forcewake_reset(dev_priv, false);
1f814dac
ID		 2362
2363 enable_rpm_wakeref_asserts(dev_priv);
2364 WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
55ec45c2 2365
bc3b9346 2366 if (intel_uncore_arm_unclaimed_mmio_detection(dev_priv))
55ec45c2
MK		 2367 DRM_ERROR("Unclaimed access detected prior to suspending\n");
2368
8a187455 2369 dev_priv->pm.suspended = true;
1fb2362b
KCA		 2370
2371 /*
c8a0bd42
PZ		 2372 * FIXME: We really should find a document that references the arguments
2373 * used below!
1fb2362b 2374 */
6f9f4b7a 2375 if (IS_BROADWELL(dev_priv)) {
d37ae19a
PZ		 2376 /*
2377 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
2378 * being detected, and the call we do at intel_runtime_resume()
2379 * won't be able to restore them. Since PCI_D3hot matches the
2380 * actual specification and appears to be working, use it.
2381 */
6f9f4b7a 2382 intel_opregion_notify_adapter(dev_priv, PCI_D3hot);
d37ae19a 2383 } else {
c8a0bd42
PZ		 2384 /*
2385 * current versions of firmware which depend on this opregion
2386 * notification have repurposed the D1 definition to mean
2387 * "runtime suspended" vs. what you would normally expect (D3)
2388 * to distinguish it from notifications that might be sent via
2389 * the suspend path.
2390 */
6f9f4b7a 2391 intel_opregion_notify_adapter(dev_priv, PCI_D1);
c8a0bd42 2392 }
8a187455 2393
59bad947 2394 assert_forcewakes_inactive(dev_priv);
dc9fb09c 2395
19625e85
L		 2396 if (!IS_VALLEYVIEW(dev_priv) || !IS_CHERRYVIEW(dev_priv))
2397 intel_hpd_poll_init(dev_priv);
2398
a8a8bd54 2399 DRM_DEBUG_KMS("Device suspended\n");
8a187455
PZ		 2400 return 0;
2401}
2402
97bea207 2403static int intel_runtime_resume(struct device *device)
8a187455
PZ		 2404{
2405 struct pci_dev *pdev = to_pci_dev(device);
2406 struct drm_device *dev = pci_get_drvdata(pdev);
fac5e23e 2407 struct drm_i915_private *dev_priv = to_i915(dev);
1a5df187 2408 int ret = 0;
8a187455 2409
604effb7
ID		 2410 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
2411 return -ENODEV;
8a187455
PZ		 2412
2413 DRM_DEBUG_KMS("Resuming device\n");
2414
1f814dac
ID		 2415 WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
2416 disable_rpm_wakeref_asserts(dev_priv);
2417
6f9f4b7a 2418 intel_opregion_notify_adapter(dev_priv, PCI_D0);
8a187455 2419 dev_priv->pm.suspended = false;
55ec45c2
MK		 2420 if (intel_uncore_unclaimed_mmio(dev_priv))
2421 DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n");
8a187455 2422
a1c41994
AD		 2423 intel_guc_resume(dev);
2424
1a5df187
PZ		 2425 if (IS_GEN6(dev_priv))
2426 intel_init_pch_refclk(dev);
31335cec 2427
507e126e
ID		 2428 if (IS_BROXTON(dev)) {
2429 bxt_disable_dc9(dev_priv);
2430 bxt_display_core_init(dev_priv, true);
f62c79b3
ID		 2431 if (dev_priv->csr.dmc_payload &&
2432 (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
2433 gen9_enable_dc5(dev_priv);
507e126e 2434 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
1a5df187 2435 hsw_disable_pc8(dev_priv);
507e126e 2436 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1a5df187 2437 ret = vlv_resume_prepare(dev_priv, true);
507e126e 2438 }
1a5df187 2439
0ab9cfeb
ID		 2440 /*
2441 * No point of rolling back things in case of an error, as the best
2442 * we can do is to hope that things will still work (and disable RPM).
2443 */
92b806d3 2444 i915_gem_init_swizzling(dev);
92b806d3 2445
b963291c 2446 intel_runtime_pm_enable_interrupts(dev_priv);
08d8a232
VS		 2447
2448 /*
2449 * On VLV/CHV display interrupts are part of the display
2450 * power well, so hpd is reinitialized from there. For
2451 * everyone else do it here.
2452 */
666a4537 2453 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
08d8a232
VS		 2454 intel_hpd_init(dev_priv);
2455
1f814dac
ID		 2456 enable_rpm_wakeref_asserts(dev_priv);
2457
0ab9cfeb
ID		 2458 if (ret)
2459 DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret);
2460 else
2461 DRM_DEBUG_KMS("Device resumed\n");
2462
2463 return ret;
8a187455
PZ		 2464}
2465
42f5551d 2466const struct dev_pm_ops i915_pm_ops = {
5545dbbf
ID		 2467 /*
2468 * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
2469 * PMSG_RESUME]
2470 */
0206e353 2471 .suspend = i915_pm_suspend,
76c4b250
ID		 2472 .suspend_late = i915_pm_suspend_late,
2473 .resume_early = i915_pm_resume_early,
0206e353 2474 .resume = i915_pm_resume,
5545dbbf
ID		 2475
2476 /*
2477 * S4 event handlers
2478 * @freeze, @freeze_late : called (1) before creating the
2479 * hibernation image [PMSG_FREEZE] and
2480 * (2) after rebooting, before restoring
2481 * the image [PMSG_QUIESCE]
2482 * @thaw, @thaw_early : called (1) after creating the hibernation
2483 * image, before writing it [PMSG_THAW]
2484 * and (2) after failing to create or
2485 * restore the image [PMSG_RECOVER]
2486 * @poweroff, @poweroff_late: called after writing the hibernation
2487 * image, before rebooting [PMSG_HIBERNATE]
2488 * @restore, @restore_early : called after rebooting and restoring the
2489 * hibernation image [PMSG_RESTORE]
2490 */
1f19ac2a
CW		 2491 .freeze = i915_pm_freeze,
2492 .freeze_late = i915_pm_freeze_late,
2493 .thaw_early = i915_pm_thaw_early,
2494 .thaw = i915_pm_thaw,
36d61e67 2495 .poweroff = i915_pm_suspend,
ab3be73f 2496 .poweroff_late = i915_pm_poweroff_late,
1f19ac2a
CW		 2497 .restore_early = i915_pm_restore_early,
2498 .restore = i915_pm_restore,
5545dbbf
ID		 2499
2500 /* S0ix (via runtime suspend) event handlers */
97bea207
PZ		 2501 .runtime_suspend = intel_runtime_suspend,
2502 .runtime_resume = intel_runtime_resume,
cbda12d7
ZW		 2503};
2504
78b68556 2505static const struct vm_operations_struct i915_gem_vm_ops = {
de151cf6 2506 .fault = i915_gem_fault,
ab00b3e5
JB		 2507 .open = drm_gem_vm_open,
2508 .close = drm_gem_vm_close,
de151cf6
JB		 2509};
2510
e08e96de
AV		 2511static const struct file_operations i915_driver_fops = {
2512 .owner = THIS_MODULE,
2513 .open = drm_open,
2514 .release = drm_release,
2515 .unlocked_ioctl = drm_ioctl,
2516 .mmap = drm_gem_mmap,
2517 .poll = drm_poll,
e08e96de
AV		 2518 .read = drm_read,
2519#ifdef CONFIG_COMPAT
2520 .compat_ioctl = i915_compat_ioctl,
2521#endif
2522 .llseek = noop_llseek,
2523};
2524
0673ad47
CW		 2525static int
2526i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data,
2527 struct drm_file *file)
2528{
2529 return -ENODEV;
2530}
2531
2532static const struct drm_ioctl_desc i915_ioctls[] = {
2533 DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2534 DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH),
2535 DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH),
2536 DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH),
2537 DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
2538 DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
2539 DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
2540 DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2541 DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
2542 DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
2543 DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2544 DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH),
2545 DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2546 DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2547 DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH),
2548 DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
2549 DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2550 DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2551 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH),
2552 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW),
2553 DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2554 DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2555 DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2556 DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
2557 DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
2558 DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2559 DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2560 DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2561 DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
2562 DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
2563 DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
2564 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
2565 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW),
2566 DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
2567 DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
2568 DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW),
2569 DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW),
2570 DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
2571 DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0),
2572 DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
2573 DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
2574 DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
2575 DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW),
2576 DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW),
2577 DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2578 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
2579 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
2580 DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
2581 DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW),
2582 DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
2583 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
2584 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
2585};
2586
1da177e4 2587static struct drm_driver driver = {
0c54781b
MW		 2588 /* Don't use MTRRs here; the Xserver or userspace app should
2589 * deal with them for Intel hardware.
792d2b9a 2590 */
673a394b 2591 .driver_features =
10ba5012 2592 DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME |
1751fcf9 2593 DRIVER_RENDER | DRIVER_MODESET,
673a394b 2594 .open = i915_driver_open,
22eae947
DA		 2595 .lastclose = i915_driver_lastclose,
2596 .preclose = i915_driver_preclose,
673a394b 2597 .postclose = i915_driver_postclose,
915b4d11 2598 .set_busid = drm_pci_set_busid,
d8e29209 2599
673a394b 2600 .gem_free_object = i915_gem_free_object,
de151cf6 2601 .gem_vm_ops = &i915_gem_vm_ops,
1286ff73
DV		 2602
2603 .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
2604 .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
2605 .gem_prime_export = i915_gem_prime_export,
2606 .gem_prime_import = i915_gem_prime_import,
2607
ff72145b 2608 .dumb_create = i915_gem_dumb_create,
da6b51d0 2609 .dumb_map_offset = i915_gem_mmap_gtt,
43387b37 2610 .dumb_destroy = drm_gem_dumb_destroy,
1da177e4 2611 .ioctls = i915_ioctls,
0673ad47 2612 .num_ioctls = ARRAY_SIZE(i915_ioctls),
e08e96de 2613 .fops = &i915_driver_fops,
22eae947
DA		 2614 .name = DRIVER_NAME,
2615 .desc = DRIVER_DESC,
2616 .date = DRIVER_DATE,
2617 .major = DRIVER_MAJOR,
2618 .minor = DRIVER_MINOR,
2619 .patchlevel = DRIVER_PATCHLEVEL,
1da177e4 2620};
ubuntu-bionic-kernel mirror