]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/gpu/drm/i915/i915_drv.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Convert sandybridge_pcode_*() to use intel_wait_for_register()
[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / i915 / i915_drv.c
CommitLineData
1da177e4
LT		 1/* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
2 */
0d6aa60b 3/*
bc54fd1a 4 *
1da177e4
LT		 5 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
6 * All Rights Reserved.
bc54fd1a
DA		 7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
23 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
0d6aa60b 28 */
1da177e4 29
e5747e3a 30#include <linux/acpi.h>
0673ad47
CW		 31#include <linux/device.h>
32#include <linux/oom.h>
e0cd3608 33#include <linux/module.h>
0673ad47
CW		 34#include <linux/pci.h>
35#include <linux/pm.h>
d6102977 36#include <linux/pm_runtime.h>
0673ad47
CW		 37#include <linux/pnp.h>
38#include <linux/slab.h>
39#include <linux/vgaarb.h>
704ab614 40#include <linux/vga_switcheroo.h>
0673ad47
CW		 41#include <linux/vt.h>
42#include <acpi/video.h>
43
44#include <drm/drmP.h>
760285e7 45#include <drm/drm_crtc_helper.h>
0673ad47
CW		 46#include <drm/i915_drm.h>
47
48#include "i915_drv.h"
49#include "i915_trace.h"
50#include "i915_vgpu.h"
51#include "intel_drv.h"
79e53945 52
112b715e
KH		 53static struct drm_driver driver;
54
0673ad47
CW		 55static unsigned int i915_load_fail_count;
56
57bool __i915_inject_load_failure(const char *func, int line)
58{
59 if (i915_load_fail_count >= i915.inject_load_failure)
60 return false;
61
62 if (++i915_load_fail_count == i915.inject_load_failure) {
63 DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n",
64 i915.inject_load_failure, func, line);
65 return true;
66 }
67
68 return false;
69}
70
71#define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI"
72#define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \
73 "providing the dmesg log by booting with drm.debug=0xf"
74
75void
76__i915_printk(struct drm_i915_private *dev_priv, const char *level,
77 const char *fmt, ...)
78{
79 static bool shown_bug_once;
80 struct device *dev = dev_priv->dev->dev;
81 bool is_error = level[1] <= KERN_ERR[1];
82 bool is_debug = level[1] == KERN_DEBUG[1];
83 struct va_format vaf;
84 va_list args;
85
86 if (is_debug && !(drm_debug & DRM_UT_DRIVER))
87 return;
88
89 va_start(args, fmt);
90
91 vaf.fmt = fmt;
92 vaf.va = &args;
93
94 dev_printk(level, dev, "[" DRM_NAME ":%ps] %pV",
95 __builtin_return_address(0), &vaf);
96
97 if (is_error && !shown_bug_once) {
98 dev_notice(dev, "%s", FDO_BUG_MSG);
99 shown_bug_once = true;
100 }
101
102 va_end(args);
103}
104
105static bool i915_error_injected(struct drm_i915_private *dev_priv)
106{
107 return i915.inject_load_failure &&
108 i915_load_fail_count == i915.inject_load_failure;
109}
110
111#define i915_load_error(dev_priv, fmt, ...) \
112 __i915_printk(dev_priv, \
113 i915_error_injected(dev_priv) ? KERN_DEBUG : KERN_ERR, \
114 fmt, ##__VA_ARGS__)
115
116
117static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
118{
119 enum intel_pch ret = PCH_NOP;
120
121 /*
122 * In a virtualized passthrough environment we can be in a
123 * setup where the ISA bridge is not able to be passed through.
124 * In this case, a south bridge can be emulated and we have to
125 * make an educated guess as to which PCH is really there.
126 */
127
128 if (IS_GEN5(dev)) {
129 ret = PCH_IBX;
130 DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n");
131 } else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
132 ret = PCH_CPT;
133 DRM_DEBUG_KMS("Assuming CouarPoint PCH\n");
134 } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
135 ret = PCH_LPT;
136 DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
137 } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
138 ret = PCH_SPT;
139 DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
140 }
141
142 return ret;
143}
144
145static void intel_detect_pch(struct drm_device *dev)
146{
147 struct drm_i915_private *dev_priv = dev->dev_private;
148 struct pci_dev *pch = NULL;
149
150 /* In all current cases, num_pipes is equivalent to the PCH_NOP setting
151 * (which really amounts to a PCH but no South Display).
152 */
153 if (INTEL_INFO(dev)->num_pipes == 0) {
154 dev_priv->pch_type = PCH_NOP;
155 return;
156 }
157
158 /*
159 * The reason to probe ISA bridge instead of Dev31:Fun0 is to
160 * make graphics device passthrough work easy for VMM, that only
161 * need to expose ISA bridge to let driver know the real hardware
162 * underneath. This is a requirement from virtualization team.
163 *
164 * In some virtualized environments (e.g. XEN), there is irrelevant
165 * ISA bridge in the system. To work reliably, we should scan trhough
166 * all the ISA bridge devices and check for the first match, instead
167 * of only checking the first one.
168 */
169 while ((pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, pch))) {
170 if (pch->vendor == PCI_VENDOR_ID_INTEL) {
171 unsigned short id = pch->device & INTEL_PCH_DEVICE_ID_MASK;
172 dev_priv->pch_id = id;
173
174 if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
175 dev_priv->pch_type = PCH_IBX;
176 DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
177 WARN_ON(!IS_GEN5(dev));
178 } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
179 dev_priv->pch_type = PCH_CPT;
180 DRM_DEBUG_KMS("Found CougarPoint PCH\n");
181 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
182 } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
183 /* PantherPoint is CPT compatible */
184 dev_priv->pch_type = PCH_CPT;
185 DRM_DEBUG_KMS("Found PantherPoint PCH\n");
186 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
187 } else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
188 dev_priv->pch_type = PCH_LPT;
189 DRM_DEBUG_KMS("Found LynxPoint PCH\n");
190 WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
191 WARN_ON(IS_HSW_ULT(dev) || IS_BDW_ULT(dev));
192 } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
193 dev_priv->pch_type = PCH_LPT;
194 DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
195 WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
196 WARN_ON(!IS_HSW_ULT(dev) && !IS_BDW_ULT(dev));
197 } else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
198 dev_priv->pch_type = PCH_SPT;
199 DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
200 WARN_ON(!IS_SKYLAKE(dev) &&
201 !IS_KABYLAKE(dev));
202 } else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
203 dev_priv->pch_type = PCH_SPT;
204 DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
205 WARN_ON(!IS_SKYLAKE(dev) &&
206 !IS_KABYLAKE(dev));
207 } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
208 (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) ||
209 ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) &&
210 pch->subsystem_vendor ==
211 PCI_SUBVENDOR_ID_REDHAT_QUMRANET &&
212 pch->subsystem_device ==
213 PCI_SUBDEVICE_ID_QEMU)) {
214 dev_priv->pch_type = intel_virt_detect_pch(dev);
215 } else
216 continue;
217
218 break;
219 }
220 }
221 if (!pch)
222 DRM_DEBUG_KMS("No PCH found.\n");
223
224 pci_dev_put(pch);
225}
226
227bool i915_semaphore_is_enabled(struct drm_i915_private *dev_priv)
228{
229 if (INTEL_GEN(dev_priv) < 6)
230 return false;
231
232 if (i915.semaphores >= 0)
233 return i915.semaphores;
234
235 /* TODO: make semaphores and Execlists play nicely together */
236 if (i915.enable_execlists)
237 return false;
238
239#ifdef CONFIG_INTEL_IOMMU
240 /* Enable semaphores on SNB when IO remapping is off */
241 if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped)
242 return false;
243#endif
244
245 return true;
246}
247
248static int i915_getparam(struct drm_device *dev, void *data,
249 struct drm_file *file_priv)
250{
251 struct drm_i915_private *dev_priv = dev->dev_private;
252 drm_i915_getparam_t *param = data;
253 int value;
254
255 switch (param->param) {
256 case I915_PARAM_IRQ_ACTIVE:
257 case I915_PARAM_ALLOW_BATCHBUFFER:
258 case I915_PARAM_LAST_DISPATCH:
259 /* Reject all old ums/dri params. */
260 return -ENODEV;
261 case I915_PARAM_CHIPSET_ID:
262 value = dev->pdev->device;
263 break;
264 case I915_PARAM_REVISION:
265 value = dev->pdev->revision;
266 break;
267 case I915_PARAM_HAS_GEM:
268 value = 1;
269 break;
270 case I915_PARAM_NUM_FENCES_AVAIL:
271 value = dev_priv->num_fence_regs;
272 break;
273 case I915_PARAM_HAS_OVERLAY:
274 value = dev_priv->overlay ? 1 : 0;
275 break;
276 case I915_PARAM_HAS_PAGEFLIPPING:
277 value = 1;
278 break;
279 case I915_PARAM_HAS_EXECBUF2:
280 /* depends on GEM */
281 value = 1;
282 break;
283 case I915_PARAM_HAS_BSD:
284 value = intel_engine_initialized(&dev_priv->engine[VCS]);
285 break;
286 case I915_PARAM_HAS_BLT:
287 value = intel_engine_initialized(&dev_priv->engine[BCS]);
288 break;
289 case I915_PARAM_HAS_VEBOX:
290 value = intel_engine_initialized(&dev_priv->engine[VECS]);
291 break;
292 case I915_PARAM_HAS_BSD2:
293 value = intel_engine_initialized(&dev_priv->engine[VCS2]);
294 break;
295 case I915_PARAM_HAS_RELAXED_FENCING:
296 value = 1;
297 break;
298 case I915_PARAM_HAS_COHERENT_RINGS:
299 value = 1;
300 break;
301 case I915_PARAM_HAS_EXEC_CONSTANTS:
302 value = INTEL_INFO(dev)->gen >= 4;
303 break;
304 case I915_PARAM_HAS_RELAXED_DELTA:
305 value = 1;
306 break;
307 case I915_PARAM_HAS_GEN7_SOL_RESET:
308 value = 1;
309 break;
310 case I915_PARAM_HAS_LLC:
311 value = HAS_LLC(dev);
312 break;
313 case I915_PARAM_HAS_WT:
314 value = HAS_WT(dev);
315 break;
316 case I915_PARAM_HAS_ALIASING_PPGTT:
317 value = USES_PPGTT(dev);
318 break;
319 case I915_PARAM_HAS_WAIT_TIMEOUT:
320 value = 1;
321 break;
322 case I915_PARAM_HAS_SEMAPHORES:
323 value = i915_semaphore_is_enabled(dev_priv);
324 break;
325 case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
326 value = 1;
327 break;
328 case I915_PARAM_HAS_SECURE_BATCHES:
329 value = capable(CAP_SYS_ADMIN);
330 break;
331 case I915_PARAM_HAS_PINNED_BATCHES:
332 value = 1;
333 break;
334 case I915_PARAM_HAS_EXEC_NO_RELOC:
335 value = 1;
336 break;
337 case I915_PARAM_HAS_EXEC_HANDLE_LUT:
338 value = 1;
339 break;
340 case I915_PARAM_CMD_PARSER_VERSION:
341 value = i915_cmd_parser_get_version(dev_priv);
342 break;
343 case I915_PARAM_HAS_COHERENT_PHYS_GTT:
344 value = 1;
345 break;
346 case I915_PARAM_MMAP_VERSION:
347 value = 1;
348 break;
349 case I915_PARAM_SUBSLICE_TOTAL:
350 value = INTEL_INFO(dev)->subslice_total;
351 if (!value)
352 return -ENODEV;
353 break;
354 case I915_PARAM_EU_TOTAL:
355 value = INTEL_INFO(dev)->eu_total;
356 if (!value)
357 return -ENODEV;
358 break;
359 case I915_PARAM_HAS_GPU_RESET:
360 value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv);
361 break;
362 case I915_PARAM_HAS_RESOURCE_STREAMER:
363 value = HAS_RESOURCE_STREAMER(dev);
364 break;
365 case I915_PARAM_HAS_EXEC_SOFTPIN:
366 value = 1;
367 break;
368 default:
369 DRM_DEBUG("Unknown parameter %d\n", param->param);
370 return -EINVAL;
371 }
372
dda33009 373 if (put_user(value, param->value))
0673ad47 374 return -EFAULT;
0673ad47
CW		 375
376 return 0;
377}
378
379static int i915_get_bridge_dev(struct drm_device *dev)
380{
381 struct drm_i915_private *dev_priv = dev->dev_private;
382
383 dev_priv->bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
384 if (!dev_priv->bridge_dev) {
385 DRM_ERROR("bridge device not found\n");
386 return -1;
387 }
388 return 0;
389}
390
391/* Allocate space for the MCH regs if needed, return nonzero on error */
392static int
393intel_alloc_mchbar_resource(struct drm_device *dev)
394{
395 struct drm_i915_private *dev_priv = dev->dev_private;
396 int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
397 u32 temp_lo, temp_hi = 0;
398 u64 mchbar_addr;
399 int ret;
400
401 if (INTEL_INFO(dev)->gen >= 4)
402 pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
403 pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
404 mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
405
406 /* If ACPI doesn't have it, assume we need to allocate it ourselves */
407#ifdef CONFIG_PNP
408 if (mchbar_addr &&
409 pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
410 return 0;
411#endif
412
413 /* Get some space for it */
414 dev_priv->mch_res.name = "i915 MCHBAR";
415 dev_priv->mch_res.flags = IORESOURCE_MEM;
416 ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,
417 &dev_priv->mch_res,
418 MCHBAR_SIZE, MCHBAR_SIZE,
419 PCIBIOS_MIN_MEM,
420 0, pcibios_align_resource,
421 dev_priv->bridge_dev);
422 if (ret) {
423 DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret);
424 dev_priv->mch_res.start = 0;
425 return ret;
426 }
427
428 if (INTEL_INFO(dev)->gen >= 4)
429 pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
430 upper_32_bits(dev_priv->mch_res.start));
431
432 pci_write_config_dword(dev_priv->bridge_dev, reg,
433 lower_32_bits(dev_priv->mch_res.start));
434 return 0;
435}
436
437/* Setup MCHBAR if possible, return true if we should disable it again */
438static void
439intel_setup_mchbar(struct drm_device *dev)
440{
441 struct drm_i915_private *dev_priv = dev->dev_private;
442 int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
443 u32 temp;
444 bool enabled;
445
446 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
447 return;
448
449 dev_priv->mchbar_need_disable = false;
450
451 if (IS_I915G(dev) || IS_I915GM(dev)) {
452 pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);
453 enabled = !!(temp & DEVEN_MCHBAR_EN);
454 } else {
455 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
456 enabled = temp & 1;
457 }
458
459 /* If it's already enabled, don't have to do anything */
460 if (enabled)
461 return;
462
463 if (intel_alloc_mchbar_resource(dev))
464 return;
465
466 dev_priv->mchbar_need_disable = true;
467
468 /* Space is allocated or reserved, so enable it. */
469 if (IS_I915G(dev) || IS_I915GM(dev)) {
470 pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
471 temp | DEVEN_MCHBAR_EN);
472 } else {
473 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
474 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);
475 }
476}
477
478static void
479intel_teardown_mchbar(struct drm_device *dev)
480{
481 struct drm_i915_private *dev_priv = dev->dev_private;
482 int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
483
484 if (dev_priv->mchbar_need_disable) {
485 if (IS_I915G(dev) || IS_I915GM(dev)) {
486 u32 deven_val;
487
488 pci_read_config_dword(dev_priv->bridge_dev, DEVEN,
489 &deven_val);
490 deven_val &= ~DEVEN_MCHBAR_EN;
491 pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
492 deven_val);
493 } else {
494 u32 mchbar_val;
495
496 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg,
497 &mchbar_val);
498 mchbar_val &= ~1;
499 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg,
500 mchbar_val);
501 }
502 }
503
504 if (dev_priv->mch_res.start)
505 release_resource(&dev_priv->mch_res);
506}
507
508/* true = enable decode, false = disable decoder */
509static unsigned int i915_vga_set_decode(void *cookie, bool state)
510{
511 struct drm_device *dev = cookie;
512
513 intel_modeset_vga_set_state(dev, state);
514 if (state)
515 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
516 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
517 else
518 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
519}
520
521static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
522{
523 struct drm_device *dev = pci_get_drvdata(pdev);
524 pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
525
526 if (state == VGA_SWITCHEROO_ON) {
527 pr_info("switched on\n");
528 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
529 /* i915 resume handler doesn't set to D0 */
530 pci_set_power_state(dev->pdev, PCI_D0);
531 i915_resume_switcheroo(dev);
532 dev->switch_power_state = DRM_SWITCH_POWER_ON;
533 } else {
534 pr_info("switched off\n");
535 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
536 i915_suspend_switcheroo(dev, pmm);
537 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
538 }
539}
540
541static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
542{
543 struct drm_device *dev = pci_get_drvdata(pdev);
544
545 /*
546 * FIXME: open_count is protected by drm_global_mutex but that would lead to
547 * locking inversion with the driver load path. And the access here is
548 * completely racy anyway. So don't bother with locking for now.
549 */
550 return dev->open_count == 0;
551}
552
553static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
554 .set_gpu_state = i915_switcheroo_set_state,
555 .reprobe = NULL,
556 .can_switch = i915_switcheroo_can_switch,
557};
558
559static void i915_gem_fini(struct drm_device *dev)
560{
561 struct drm_i915_private *dev_priv = to_i915(dev);
562
563 /*
564 * Neither the BIOS, ourselves or any other kernel
565 * expects the system to be in execlists mode on startup,
566 * so we need to reset the GPU back to legacy mode. And the only
567 * known way to disable logical contexts is through a GPU reset.
568 *
569 * So in order to leave the system in a known default configuration,
570 * always reset the GPU upon unload. Afterwards we then clean up the
571 * GEM state tracking, flushing off the requests and leaving the
572 * system in a known idle state.
573 *
574 * Note that is of the upmost importance that the GPU is idle and
575 * all stray writes are flushed *before* we dismantle the backing
576 * storage for the pinned objects.
577 *
578 * However, since we are uncertain that reseting the GPU on older
579 * machines is a good idea, we don't - just in case it leaves the
580 * machine in an unusable condition.
581 */
582 if (HAS_HW_CONTEXTS(dev)) {
583 int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
584 WARN_ON(reset && reset != -ENODEV);
585 }
586
587 mutex_lock(&dev->struct_mutex);
588 i915_gem_reset(dev);
589 i915_gem_cleanup_engines(dev);
590 i915_gem_context_fini(dev);
591 mutex_unlock(&dev->struct_mutex);
592
593 WARN_ON(!list_empty(&to_i915(dev)->context_list));
594}
595
596static int i915_load_modeset_init(struct drm_device *dev)
597{
598 struct drm_i915_private *dev_priv = dev->dev_private;
599 int ret;
600
601 if (i915_inject_load_failure())
602 return -ENODEV;
603
604 ret = intel_bios_init(dev_priv);
605 if (ret)
606 DRM_INFO("failed to find VBIOS tables\n");
607
608 /* If we have > 1 VGA cards, then we need to arbitrate access
609 * to the common VGA resources.
610 *
611 * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
612 * then we do not take part in VGA arbitration and the
613 * vga_client_register() fails with -ENODEV.
614 */
615 ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode);
616 if (ret && ret != -ENODEV)
617 goto out;
618
619 intel_register_dsm_handler();
620
621 ret = vga_switcheroo_register_client(dev->pdev, &i915_switcheroo_ops, false);
622 if (ret)
623 goto cleanup_vga_client;
624
625 /* must happen before intel_power_domains_init_hw() on VLV/CHV */
626 intel_update_rawclk(dev_priv);
627
628 intel_power_domains_init_hw(dev_priv, false);
629
630 intel_csr_ucode_init(dev_priv);
631
632 ret = intel_irq_install(dev_priv);
633 if (ret)
634 goto cleanup_csr;
635
636 intel_setup_gmbus(dev);
637
638 /* Important: The output setup functions called by modeset_init need
639 * working irqs for e.g. gmbus and dp aux transfers. */
640 intel_modeset_init(dev);
641
642 intel_guc_init(dev);
643
644 ret = i915_gem_init(dev);
645 if (ret)
646 goto cleanup_irq;
647
648 intel_modeset_gem_init(dev);
649
650 if (INTEL_INFO(dev)->num_pipes == 0)
651 return 0;
652
653 ret = intel_fbdev_init(dev);
654 if (ret)
655 goto cleanup_gem;
656
657 /* Only enable hotplug handling once the fbdev is fully set up. */
658 intel_hpd_init(dev_priv);
659
660 drm_kms_helper_poll_init(dev);
661
662 return 0;
663
664cleanup_gem:
665 i915_gem_fini(dev);
666cleanup_irq:
667 intel_guc_fini(dev);
668 drm_irq_uninstall(dev);
669 intel_teardown_gmbus(dev);
670cleanup_csr:
671 intel_csr_ucode_fini(dev_priv);
672 intel_power_domains_fini(dev_priv);
673 vga_switcheroo_unregister_client(dev->pdev);
674cleanup_vga_client:
675 vga_client_register(dev->pdev, NULL, NULL, NULL);
676out:
677 return ret;
678}
679
680#if IS_ENABLED(CONFIG_FB)
681static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
682{
683 struct apertures_struct *ap;
684 struct pci_dev *pdev = dev_priv->dev->pdev;
685 struct i915_ggtt *ggtt = &dev_priv->ggtt;
686 bool primary;
687 int ret;
688
689 ap = alloc_apertures(1);
690 if (!ap)
691 return -ENOMEM;
692
693 ap->ranges[0].base = ggtt->mappable_base;
694 ap->ranges[0].size = ggtt->mappable_end;
695
696 primary =
697 pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
698
699 ret = remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
700
701 kfree(ap);
702
703 return ret;
704}
705#else
706static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
707{
708 return 0;
709}
710#endif
711
712#if !defined(CONFIG_VGA_CONSOLE)
713static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
714{
715 return 0;
716}
717#elif !defined(CONFIG_DUMMY_CONSOLE)
718static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
719{
720 return -ENODEV;
721}
722#else
723static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
724{
725 int ret = 0;
726
727 DRM_INFO("Replacing VGA console driver\n");
728
729 console_lock();
730 if (con_is_bound(&vga_con))
731 ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1);
732 if (ret == 0) {
733 ret = do_unregister_con_driver(&vga_con);
734
735 /* Ignore "already unregistered". */
736 if (ret == -ENODEV)
737 ret = 0;
738 }
739 console_unlock();
740
741 return ret;
742}
743#endif
744
745static void i915_dump_device_info(struct drm_i915_private *dev_priv)
746{
747 const struct intel_device_info *info = &dev_priv->info;
748
749#define PRINT_S(name) "%s"
750#define SEP_EMPTY
751#define PRINT_FLAG(name) info->name ? #name "," : ""
752#define SEP_COMMA ,
753 DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x rev=0x%02x flags="
754 DEV_INFO_FOR_EACH_FLAG(PRINT_S, SEP_EMPTY),
755 info->gen,
756 dev_priv->dev->pdev->device,
757 dev_priv->dev->pdev->revision,
758 DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_COMMA));
759#undef PRINT_S
760#undef SEP_EMPTY
761#undef PRINT_FLAG
762#undef SEP_COMMA
763}
764
765static void cherryview_sseu_info_init(struct drm_device *dev)
766{
767 struct drm_i915_private *dev_priv = dev->dev_private;
768 struct intel_device_info *info;
769 u32 fuse, eu_dis;
770
771 info = (struct intel_device_info *)&dev_priv->info;
772 fuse = I915_READ(CHV_FUSE_GT);
773
774 info->slice_total = 1;
775
776 if (!(fuse & CHV_FGT_DISABLE_SS0)) {
777 info->subslice_per_slice++;
778 eu_dis = fuse & (CHV_FGT_EU_DIS_SS0_R0_MASK |
779 CHV_FGT_EU_DIS_SS0_R1_MASK);
780 info->eu_total += 8 - hweight32(eu_dis);
781 }
782
783 if (!(fuse & CHV_FGT_DISABLE_SS1)) {
784 info->subslice_per_slice++;
785 eu_dis = fuse & (CHV_FGT_EU_DIS_SS1_R0_MASK |
786 CHV_FGT_EU_DIS_SS1_R1_MASK);
787 info->eu_total += 8 - hweight32(eu_dis);
788 }
789
790 info->subslice_total = info->subslice_per_slice;
791 /*
792 * CHV expected to always have a uniform distribution of EU
793 * across subslices.
794 */
795 info->eu_per_subslice = info->subslice_total ?
796 info->eu_total / info->subslice_total :
797 0;
798 /*
799 * CHV supports subslice power gating on devices with more than
800 * one subslice, and supports EU power gating on devices with
801 * more than one EU pair per subslice.
802 */
803 info->has_slice_pg = 0;
804 info->has_subslice_pg = (info->subslice_total > 1);
805 info->has_eu_pg = (info->eu_per_subslice > 2);
806}
807
808static void gen9_sseu_info_init(struct drm_device *dev)
809{
810 struct drm_i915_private *dev_priv = dev->dev_private;
811 struct intel_device_info *info;
812 int s_max = 3, ss_max = 4, eu_max = 8;
813 int s, ss;
814 u32 fuse2, s_enable, ss_disable, eu_disable;
815 u8 eu_mask = 0xff;
816
817 info = (struct intel_device_info *)&dev_priv->info;
818 fuse2 = I915_READ(GEN8_FUSE2);
819 s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >>
820 GEN8_F2_S_ENA_SHIFT;
821 ss_disable = (fuse2 & GEN9_F2_SS_DIS_MASK) >>
822 GEN9_F2_SS_DIS_SHIFT;
823
824 info->slice_total = hweight32(s_enable);
825 /*
826 * The subslice disable field is global, i.e. it applies
827 * to each of the enabled slices.
828 */
829 info->subslice_per_slice = ss_max - hweight32(ss_disable);
830 info->subslice_total = info->slice_total *
831 info->subslice_per_slice;
832
833 /*
834 * Iterate through enabled slices and subslices to
835 * count the total enabled EU.
836 */
837 for (s = 0; s < s_max; s++) {
838 if (!(s_enable & (0x1 << s)))
839 /* skip disabled slice */
840 continue;
841
842 eu_disable = I915_READ(GEN9_EU_DISABLE(s));
843 for (ss = 0; ss < ss_max; ss++) {
844 int eu_per_ss;
845
846 if (ss_disable & (0x1 << ss))
847 /* skip disabled subslice */
848 continue;
849
850 eu_per_ss = eu_max - hweight8((eu_disable >> (ss*8)) &
851 eu_mask);
852
853 /*
854 * Record which subslice(s) has(have) 7 EUs. we
855 * can tune the hash used to spread work among
856 * subslices if they are unbalanced.
857 */
858 if (eu_per_ss == 7)
859 info->subslice_7eu[s] |= 1 << ss;
860
861 info->eu_total += eu_per_ss;
862 }
863 }
864
865 /*
866 * SKL is expected to always have a uniform distribution
867 * of EU across subslices with the exception that any one
868 * EU in any one subslice may be fused off for die
869 * recovery. BXT is expected to be perfectly uniform in EU
870 * distribution.
871 */
872 info->eu_per_subslice = info->subslice_total ?
873 DIV_ROUND_UP(info->eu_total,
874 info->subslice_total) : 0;
875 /*
876 * SKL supports slice power gating on devices with more than
877 * one slice, and supports EU power gating on devices with
878 * more than one EU pair per subslice. BXT supports subslice
879 * power gating on devices with more than one subslice, and
880 * supports EU power gating on devices with more than one EU
881 * pair per subslice.
882 */
883 info->has_slice_pg = ((IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) &&
884 (info->slice_total > 1));
885 info->has_subslice_pg = (IS_BROXTON(dev) && (info->subslice_total > 1));
886 info->has_eu_pg = (info->eu_per_subslice > 2);
887
888 if (IS_BROXTON(dev)) {
889#define IS_SS_DISABLED(_ss_disable, ss) (_ss_disable & (0x1 << ss))
890 /*
891 * There is a HW issue in 2x6 fused down parts that requires
892 * Pooled EU to be enabled as a WA. The pool configuration
893 * changes depending upon which subslice is fused down. This
894 * doesn't affect if the device has all 3 subslices enabled.
895 */
896 /* WaEnablePooledEuFor2x6:bxt */
897 info->has_pooled_eu = ((info->subslice_per_slice == 3) ||
898 (info->subslice_per_slice == 2 &&
899 INTEL_REVID(dev) < BXT_REVID_C0));
900
901 info->min_eu_in_pool = 0;
902 if (info->has_pooled_eu) {
903 if (IS_SS_DISABLED(ss_disable, 0) ||
904 IS_SS_DISABLED(ss_disable, 2))
905 info->min_eu_in_pool = 3;
906 else if (IS_SS_DISABLED(ss_disable, 1))
907 info->min_eu_in_pool = 6;
908 else
909 info->min_eu_in_pool = 9;
910 }
911#undef IS_SS_DISABLED
912 }
913}
914
915static void broadwell_sseu_info_init(struct drm_device *dev)
916{
917 struct drm_i915_private *dev_priv = dev->dev_private;
918 struct intel_device_info *info;
919 const int s_max = 3, ss_max = 3, eu_max = 8;
920 int s, ss;
921 u32 fuse2, eu_disable[s_max], s_enable, ss_disable;
922
923 fuse2 = I915_READ(GEN8_FUSE2);
924 s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
925 ss_disable = (fuse2 & GEN8_F2_SS_DIS_MASK) >> GEN8_F2_SS_DIS_SHIFT;
926
927 eu_disable[0] = I915_READ(GEN8_EU_DISABLE0) & GEN8_EU_DIS0_S0_MASK;
928 eu_disable[1] = (I915_READ(GEN8_EU_DISABLE0) >> GEN8_EU_DIS0_S1_SHIFT) |
929 ((I915_READ(GEN8_EU_DISABLE1) & GEN8_EU_DIS1_S1_MASK) <<
930 (32 - GEN8_EU_DIS0_S1_SHIFT));
931 eu_disable[2] = (I915_READ(GEN8_EU_DISABLE1) >> GEN8_EU_DIS1_S2_SHIFT) |
932 ((I915_READ(GEN8_EU_DISABLE2) & GEN8_EU_DIS2_S2_MASK) <<
933 (32 - GEN8_EU_DIS1_S2_SHIFT));
934
935
936 info = (struct intel_device_info *)&dev_priv->info;
937 info->slice_total = hweight32(s_enable);
938
939 /*
940 * The subslice disable field is global, i.e. it applies
941 * to each of the enabled slices.
942 */
943 info->subslice_per_slice = ss_max - hweight32(ss_disable);
944 info->subslice_total = info->slice_total * info->subslice_per_slice;
945
946 /*
947 * Iterate through enabled slices and subslices to
948 * count the total enabled EU.
949 */
950 for (s = 0; s < s_max; s++) {
951 if (!(s_enable & (0x1 << s)))
952 /* skip disabled slice */
953 continue;
954
955 for (ss = 0; ss < ss_max; ss++) {
956 u32 n_disabled;
957
958 if (ss_disable & (0x1 << ss))
959 /* skip disabled subslice */
960 continue;
961
962 n_disabled = hweight8(eu_disable[s] >> (ss * eu_max));
963
964 /*
965 * Record which subslices have 7 EUs.
966 */
967 if (eu_max - n_disabled == 7)
968 info->subslice_7eu[s] |= 1 << ss;
969
970 info->eu_total += eu_max - n_disabled;
971 }
972 }
973
974 /*
975 * BDW is expected to always have a uniform distribution of EU across
976 * subslices with the exception that any one EU in any one subslice may
977 * be fused off for die recovery.
978 */
979 info->eu_per_subslice = info->subslice_total ?
980 DIV_ROUND_UP(info->eu_total, info->subslice_total) : 0;
981
982 /*
983 * BDW supports slice power gating on devices with more than
984 * one slice.
985 */
986 info->has_slice_pg = (info->slice_total > 1);
987 info->has_subslice_pg = 0;
988 info->has_eu_pg = 0;
989}
990
991/*
992 * Determine various intel_device_info fields at runtime.
993 *
994 * Use it when either:
995 * - it's judged too laborious to fill n static structures with the limit
996 * when a simple if statement does the job,
997 * - run-time checks (eg read fuse/strap registers) are needed.
998 *
999 * This function needs to be called:
1000 * - after the MMIO has been setup as we are reading registers,
1001 * - after the PCH has been detected,
1002 * - before the first usage of the fields it can tweak.
1003 */
1004static void intel_device_info_runtime_init(struct drm_device *dev)
1005{
1006 struct drm_i915_private *dev_priv = dev->dev_private;
1007 struct intel_device_info *info;
1008 enum pipe pipe;
1009
1010 info = (struct intel_device_info *)&dev_priv->info;
1011
1012 /*
1013 * Skylake and Broxton currently don't expose the topmost plane as its
1014 * use is exclusive with the legacy cursor and we only want to expose
1015 * one of those, not both. Until we can safely expose the topmost plane
1016 * as a DRM_PLANE_TYPE_CURSOR with all the features exposed/supported,
1017 * we don't expose the topmost plane at all to prevent ABI breakage
1018 * down the line.
1019 */
1020 if (IS_BROXTON(dev)) {
1021 info->num_sprites[PIPE_A] = 2;
1022 info->num_sprites[PIPE_B] = 2;
1023 info->num_sprites[PIPE_C] = 1;
1024 } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
1025 for_each_pipe(dev_priv, pipe)
1026 info->num_sprites[pipe] = 2;
1027 else
1028 for_each_pipe(dev_priv, pipe)
1029 info->num_sprites[pipe] = 1;
1030
1031 if (i915.disable_display) {
1032 DRM_INFO("Display disabled (module parameter)\n");
1033 info->num_pipes = 0;
1034 } else if (info->num_pipes > 0 &&
1035 (IS_GEN7(dev_priv) || IS_GEN8(dev_priv)) &&
1036 HAS_PCH_SPLIT(dev)) {
1037 u32 fuse_strap = I915_READ(FUSE_STRAP);
1038 u32 sfuse_strap = I915_READ(SFUSE_STRAP);
1039
1040 /*
1041 * SFUSE_STRAP is supposed to have a bit signalling the display
1042 * is fused off. Unfortunately it seems that, at least in
1043 * certain cases, fused off display means that PCH display
1044 * reads don't land anywhere. In that case, we read 0s.
1045 *
1046 * On CPT/PPT, we can detect this case as SFUSE_STRAP_FUSE_LOCK
1047 * should be set when taking over after the firmware.
1048 */
1049 if (fuse_strap & ILK_INTERNAL_DISPLAY_DISABLE ||
1050 sfuse_strap & SFUSE_STRAP_DISPLAY_DISABLED ||
1051 (dev_priv->pch_type == PCH_CPT &&
1052 !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) {
1053 DRM_INFO("Display fused off, disabling\n");
1054 info->num_pipes = 0;
1055 } else if (fuse_strap & IVB_PIPE_C_DISABLE) {
1056 DRM_INFO("PipeC fused off\n");
1057 info->num_pipes -= 1;
1058 }
1059 } else if (info->num_pipes > 0 && IS_GEN9(dev_priv)) {
1060 u32 dfsm = I915_READ(SKL_DFSM);
1061 u8 disabled_mask = 0;
1062 bool invalid;
1063 int num_bits;
1064
1065 if (dfsm & SKL_DFSM_PIPE_A_DISABLE)
1066 disabled_mask |= BIT(PIPE_A);
1067 if (dfsm & SKL_DFSM_PIPE_B_DISABLE)
1068 disabled_mask |= BIT(PIPE_B);
1069 if (dfsm & SKL_DFSM_PIPE_C_DISABLE)
1070 disabled_mask |= BIT(PIPE_C);
1071
1072 num_bits = hweight8(disabled_mask);
1073
1074 switch (disabled_mask) {
1075 case BIT(PIPE_A):
1076 case BIT(PIPE_B):
1077 case BIT(PIPE_A) | BIT(PIPE_B):
1078 case BIT(PIPE_A) | BIT(PIPE_C):
1079 invalid = true;
1080 break;
1081 default:
1082 invalid = false;
1083 }
1084
1085 if (num_bits > info->num_pipes || invalid)
1086 DRM_ERROR("invalid pipe fuse configuration: 0x%x\n",
1087 disabled_mask);
1088 else
1089 info->num_pipes -= num_bits;
1090 }
1091
1092 /* Initialize slice/subslice/EU info */
1093 if (IS_CHERRYVIEW(dev))
1094 cherryview_sseu_info_init(dev);
1095 else if (IS_BROADWELL(dev))
1096 broadwell_sseu_info_init(dev);
1097 else if (INTEL_INFO(dev)->gen >= 9)
1098 gen9_sseu_info_init(dev);
1099
1100 info->has_snoop = !info->has_llc;
1101
1102 /* Snooping is broken on BXT A stepping. */
1103 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
1104 info->has_snoop = false;
1105
1106 DRM_DEBUG_DRIVER("slice total: %u\n", info->slice_total);
1107 DRM_DEBUG_DRIVER("subslice total: %u\n", info->subslice_total);
1108 DRM_DEBUG_DRIVER("subslice per slice: %u\n", info->subslice_per_slice);
1109 DRM_DEBUG_DRIVER("EU total: %u\n", info->eu_total);
1110 DRM_DEBUG_DRIVER("EU per subslice: %u\n", info->eu_per_subslice);
1111 DRM_DEBUG_DRIVER("has slice power gating: %s\n",
1112 info->has_slice_pg ? "y" : "n");
1113 DRM_DEBUG_DRIVER("has subslice power gating: %s\n",
1114 info->has_subslice_pg ? "y" : "n");
1115 DRM_DEBUG_DRIVER("has EU power gating: %s\n",
1116 info->has_eu_pg ? "y" : "n");
1117
1118 i915.enable_execlists =
1119 intel_sanitize_enable_execlists(dev_priv,
1120 i915.enable_execlists);
1121
1122 /*
1123 * i915.enable_ppgtt is read-only, so do an early pass to validate the
1124 * user's requested state against the hardware/driver capabilities. We
1125 * do this now so that we can print out any log messages once rather
1126 * than every time we check intel_enable_ppgtt().
1127 */
1128 i915.enable_ppgtt =
1129 intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt);
1130 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
1131}
1132
1133static void intel_init_dpio(struct drm_i915_private *dev_priv)
1134{
1135 /*
1136 * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
1137 * CHV x1 PHY (DP/HDMI D)
1138 * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
1139 */
1140 if (IS_CHERRYVIEW(dev_priv)) {
1141 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
1142 DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
1143 } else if (IS_VALLEYVIEW(dev_priv)) {
1144 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
1145 }
1146}
1147
1148static int i915_workqueues_init(struct drm_i915_private *dev_priv)
1149{
1150 /*
1151 * The i915 workqueue is primarily used for batched retirement of
1152 * requests (and thus managing bo) once the task has been completed
1153 * by the GPU. i915_gem_retire_requests() is called directly when we
1154 * need high-priority retirement, such as waiting for an explicit
1155 * bo.
1156 *
1157 * It is also used for periodic low-priority events, such as
1158 * idle-timers and recording error state.
1159 *
1160 * All tasks on the workqueue are expected to acquire the dev mutex
1161 * so there is no point in running more than one instance of the
1162 * workqueue at any time. Use an ordered one.
1163 */
1164 dev_priv->wq = alloc_ordered_workqueue("i915", 0);
1165 if (dev_priv->wq == NULL)
1166 goto out_err;
1167
1168 dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
1169 if (dev_priv->hotplug.dp_wq == NULL)
1170 goto out_free_wq;
1171
1172 dev_priv->gpu_error.hangcheck_wq =
1173 alloc_ordered_workqueue("i915-hangcheck", 0);
1174 if (dev_priv->gpu_error.hangcheck_wq == NULL)
1175 goto out_free_dp_wq;
1176
1177 return 0;
1178
1179out_free_dp_wq:
1180 destroy_workqueue(dev_priv->hotplug.dp_wq);
1181out_free_wq:
1182 destroy_workqueue(dev_priv->wq);
1183out_err:
1184 DRM_ERROR("Failed to allocate workqueues.\n");
1185
1186 return -ENOMEM;
1187}
1188
1189static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
1190{
1191 destroy_workqueue(dev_priv->gpu_error.hangcheck_wq);
1192 destroy_workqueue(dev_priv->hotplug.dp_wq);
1193 destroy_workqueue(dev_priv->wq);
1194}
1195
1196/**
1197 * i915_driver_init_early - setup state not requiring device access
1198 * @dev_priv: device private
1199 *
1200 * Initialize everything that is a "SW-only" state, that is state not
1201 * requiring accessing the device or exposing the driver via kernel internal
1202 * or userspace interfaces. Example steps belonging here: lock initialization,
1203 * system memory allocation, setting up device specific attributes and
1204 * function hooks not requiring accessing the device.
1205 */
1206static int i915_driver_init_early(struct drm_i915_private *dev_priv,
1207 const struct pci_device_id *ent)
1208{
1209 const struct intel_device_info *match_info =
1210 (struct intel_device_info *)ent->driver_data;
1211 struct intel_device_info *device_info;
1212 int ret = 0;
1213
1214 if (i915_inject_load_failure())
1215 return -ENODEV;
1216
1217 /* Setup the write-once "constant" device info */
1218 device_info = (struct intel_device_info *)&dev_priv->info;
1219 memcpy(device_info, match_info, sizeof(*device_info));
1220 device_info->device_id = dev_priv->drm.pdev->device;
1221
1222 BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE);
1223 device_info->gen_mask = BIT(device_info->gen - 1);
1224
1225 spin_lock_init(&dev_priv->irq_lock);
1226 spin_lock_init(&dev_priv->gpu_error.lock);
1227 mutex_init(&dev_priv->backlight_lock);
1228 spin_lock_init(&dev_priv->uncore.lock);
1229 spin_lock_init(&dev_priv->mm.object_stat_lock);
1230 spin_lock_init(&dev_priv->mmio_flip_lock);
1231 mutex_init(&dev_priv->sb_lock);
1232 mutex_init(&dev_priv->modeset_restore_lock);
1233 mutex_init(&dev_priv->av_mutex);
1234 mutex_init(&dev_priv->wm.wm_mutex);
1235 mutex_init(&dev_priv->pps_mutex);
1236
1237 ret = i915_workqueues_init(dev_priv);
1238 if (ret < 0)
1239 return ret;
1240
1241 ret = intel_gvt_init(dev_priv);
1242 if (ret < 0)
1243 goto err_workqueues;
1244
1245 /* This must be called before any calls to HAS_PCH_* */
1246 intel_detect_pch(&dev_priv->drm);
1247
1248 intel_pm_setup(&dev_priv->drm);
1249 intel_init_dpio(dev_priv);
1250 intel_power_domains_init(dev_priv);
1251 intel_irq_init(dev_priv);
1252 intel_init_display_hooks(dev_priv);
1253 intel_init_clock_gating_hooks(dev_priv);
1254 intel_init_audio_hooks(dev_priv);
1255 i915_gem_load_init(&dev_priv->drm);
1256
1257 intel_display_crc_init(&dev_priv->drm);
1258
1259 i915_dump_device_info(dev_priv);
1260
1261 /* Not all pre-production machines fall into this category, only the
1262 * very first ones. Almost everything should work, except for maybe
1263 * suspend/resume. And we don't implement workarounds that affect only
1264 * pre-production machines. */
1265 if (IS_HSW_EARLY_SDV(dev_priv))
1266 DRM_INFO("This is an early pre-production Haswell machine. "
1267 "It may not be fully functional.\n");
1268
1269 return 0;
1270
1271err_workqueues:
1272 i915_workqueues_cleanup(dev_priv);
1273 return ret;
1274}
1275
1276/**
1277 * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early()
1278 * @dev_priv: device private
1279 */
1280static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
1281{
1282 i915_gem_load_cleanup(dev_priv->dev);
1283 i915_workqueues_cleanup(dev_priv);
1284}
1285
1286static int i915_mmio_setup(struct drm_device *dev)
1287{
1288 struct drm_i915_private *dev_priv = to_i915(dev);
1289 int mmio_bar;
1290 int mmio_size;
1291
1292 mmio_bar = IS_GEN2(dev) ? 1 : 0;
1293 /*
1294 * Before gen4, the registers and the GTT are behind different BARs.
1295 * However, from gen4 onwards, the registers and the GTT are shared
1296 * in the same BAR, so we want to restrict this ioremap from
1297 * clobbering the GTT which we want ioremap_wc instead. Fortunately,
1298 * the register BAR remains the same size for all the earlier
1299 * generations up to Ironlake.
1300 */
1301 if (INTEL_INFO(dev)->gen < 5)
1302 mmio_size = 512 * 1024;
1303 else
1304 mmio_size = 2 * 1024 * 1024;
1305 dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, mmio_size);
1306 if (dev_priv->regs == NULL) {
1307 DRM_ERROR("failed to map registers\n");
1308
1309 return -EIO;
1310 }
1311
1312 /* Try to make sure MCHBAR is enabled before poking at it */
1313 intel_setup_mchbar(dev);
1314
1315 return 0;
1316}
1317
1318static void i915_mmio_cleanup(struct drm_device *dev)
1319{
1320 struct drm_i915_private *dev_priv = to_i915(dev);
1321
1322 intel_teardown_mchbar(dev);
1323 pci_iounmap(dev->pdev, dev_priv->regs);
1324}
1325
1326/**
1327 * i915_driver_init_mmio - setup device MMIO
1328 * @dev_priv: device private
1329 *
1330 * Setup minimal device state necessary for MMIO accesses later in the
1331 * initialization sequence. The setup here should avoid any other device-wide
1332 * side effects or exposing the driver via kernel internal or user space
1333 * interfaces.
1334 */
1335static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
1336{
1337 struct drm_device *dev = dev_priv->dev;
1338 int ret;
1339
1340 if (i915_inject_load_failure())
1341 return -ENODEV;
1342
1343 if (i915_get_bridge_dev(dev))
1344 return -EIO;
1345
1346 ret = i915_mmio_setup(dev);
1347 if (ret < 0)
1348 goto put_bridge;
1349
1350 intel_uncore_init(dev_priv);
1351
1352 return 0;
1353
1354put_bridge:
1355 pci_dev_put(dev_priv->bridge_dev);
1356
1357 return ret;
1358}
1359
1360/**
1361 * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio()
1362 * @dev_priv: device private
1363 */
1364static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv)
1365{
1366 struct drm_device *dev = dev_priv->dev;
1367
1368 intel_uncore_fini(dev_priv);
1369 i915_mmio_cleanup(dev);
1370 pci_dev_put(dev_priv->bridge_dev);
1371}
1372
1373/**
1374 * i915_driver_init_hw - setup state requiring device access
1375 * @dev_priv: device private
1376 *
1377 * Setup state that requires accessing the device, but doesn't require
1378 * exposing the driver via kernel internal or userspace interfaces.
1379 */
1380static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
1381{
1382 struct drm_device *dev = dev_priv->dev;
1383 struct i915_ggtt *ggtt = &dev_priv->ggtt;
1384 uint32_t aperture_size;
1385 int ret;
1386
1387 if (i915_inject_load_failure())
1388 return -ENODEV;
1389
1390 intel_device_info_runtime_init(dev);
1391
1392 ret = i915_ggtt_init_hw(dev);
1393 if (ret)
1394 return ret;
1395
1396 ret = i915_ggtt_enable_hw(dev);
1397 if (ret) {
1398 DRM_ERROR("failed to enable GGTT\n");
1399 goto out_ggtt;
1400 }
1401
1402 /* WARNING: Apparently we must kick fbdev drivers before vgacon,
1403 * otherwise the vga fbdev driver falls over. */
1404 ret = i915_kick_out_firmware_fb(dev_priv);
1405 if (ret) {
1406 DRM_ERROR("failed to remove conflicting framebuffer drivers\n");
1407 goto out_ggtt;
1408 }
1409
1410 ret = i915_kick_out_vgacon(dev_priv);
1411 if (ret) {
1412 DRM_ERROR("failed to remove conflicting VGA console\n");
1413 goto out_ggtt;
1414 }
1415
1416 pci_set_master(dev->pdev);
1417
1418 /* overlay on gen2 is broken and can't address above 1G */
1419 if (IS_GEN2(dev)) {
1420 ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
1421 if (ret) {
1422 DRM_ERROR("failed to set DMA mask\n");
1423
1424 goto out_ggtt;
1425 }
1426 }
1427
1428
1429 /* 965GM sometimes incorrectly writes to hardware status page (HWS)
1430 * using 32bit addressing, overwriting memory if HWS is located
1431 * above 4GB.
1432 *
1433 * The documentation also mentions an issue with undefined
1434 * behaviour if any general state is accessed within a page above 4GB,
1435 * which also needs to be handled carefully.
1436 */
1437 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) {
1438 ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32));
1439
1440 if (ret) {
1441 DRM_ERROR("failed to set DMA mask\n");
1442
1443 goto out_ggtt;
1444 }
1445 }
1446
1447 aperture_size = ggtt->mappable_end;
1448
1449 ggtt->mappable =
1450 io_mapping_create_wc(ggtt->mappable_base,
1451 aperture_size);
1452 if (!ggtt->mappable) {
1453 ret = -EIO;
1454 goto out_ggtt;
1455 }
1456
1457 ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base,
1458 aperture_size);
1459
1460 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
1461 PM_QOS_DEFAULT_VALUE);
1462
1463 intel_uncore_sanitize(dev_priv);
1464
1465 intel_opregion_setup(dev_priv);
1466
1467 i915_gem_load_init_fences(dev_priv);
1468
1469 /* On the 945G/GM, the chipset reports the MSI capability on the
1470 * integrated graphics even though the support isn't actually there
1471 * according to the published specs. It doesn't appear to function
1472 * correctly in testing on 945G.
1473 * This may be a side effect of MSI having been made available for PEG
1474 * and the registers being closely associated.
1475 *
1476 * According to chipset errata, on the 965GM, MSI interrupts may
1477 * be lost or delayed, but we use them anyways to avoid
1478 * stuck interrupts on some machines.
1479 */
1480 if (!IS_I945G(dev) && !IS_I945GM(dev)) {
1481 if (pci_enable_msi(dev->pdev) < 0)
1482 DRM_DEBUG_DRIVER("can't enable MSI");
1483 }
1484
1485 return 0;
1486
1487out_ggtt:
1488 i915_ggtt_cleanup_hw(dev);
1489
1490 return ret;
1491}
1492
1493/**
1494 * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw()
1495 * @dev_priv: device private
1496 */
1497static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv)
1498{
1499 struct drm_device *dev = dev_priv->dev;
1500 struct i915_ggtt *ggtt = &dev_priv->ggtt;
1501
1502 if (dev->pdev->msi_enabled)
1503 pci_disable_msi(dev->pdev);
1504
1505 pm_qos_remove_request(&dev_priv->pm_qos);
1506 arch_phys_wc_del(ggtt->mtrr);
1507 io_mapping_free(ggtt->mappable);
1508 i915_ggtt_cleanup_hw(dev);
1509}
1510
1511/**
1512 * i915_driver_register - register the driver with the rest of the system
1513 * @dev_priv: device private
1514 *
1515 * Perform any steps necessary to make the driver available via kernel
1516 * internal or userspace interfaces.
1517 */
1518static void i915_driver_register(struct drm_i915_private *dev_priv)
1519{
1520 struct drm_device *dev = dev_priv->dev;
1521
1522 i915_gem_shrinker_init(dev_priv);
1523
1524 /*
1525 * Notify a valid surface after modesetting,
1526 * when running inside a VM.
1527 */
1528 if (intel_vgpu_active(dev_priv))
1529 I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY);
1530
1531 /* Reveal our presence to userspace */
1532 if (drm_dev_register(dev, 0) == 0) {
1533 i915_debugfs_register(dev_priv);
1534 i915_setup_sysfs(dev);
1535 } else
1536 DRM_ERROR("Failed to register driver for userspace access!\n");
1537
1538 if (INTEL_INFO(dev_priv)->num_pipes) {
1539 /* Must be done after probing outputs */
1540 intel_opregion_register(dev_priv);
1541 acpi_video_register();
1542 }
1543
1544 if (IS_GEN5(dev_priv))
1545 intel_gpu_ips_init(dev_priv);
1546
1547 i915_audio_component_init(dev_priv);
1548
1549 /*
1550 * Some ports require correctly set-up hpd registers for detection to
1551 * work properly (leading to ghost connected connector status), e.g. VGA
1552 * on gm45. Hence we can only set up the initial fbdev config after hpd
1553 * irqs are fully enabled. We do it last so that the async config
1554 * cannot run before the connectors are registered.
1555 */
1556 intel_fbdev_initial_config_async(dev);
1557}
1558
1559/**
1560 * i915_driver_unregister - cleanup the registration done in i915_driver_regiser()
1561 * @dev_priv: device private
1562 */
1563static void i915_driver_unregister(struct drm_i915_private *dev_priv)
1564{
1565 i915_audio_component_cleanup(dev_priv);
1566
1567 intel_gpu_ips_teardown();
1568 acpi_video_unregister();
1569 intel_opregion_unregister(dev_priv);
1570
1571 i915_teardown_sysfs(dev_priv->dev);
1572 i915_debugfs_unregister(dev_priv);
1573 drm_dev_unregister(dev_priv->dev);
1574
1575 i915_gem_shrinker_cleanup(dev_priv);
1576}
1577
1578/**
1579 * i915_driver_load - setup chip and create an initial config
1580 * @dev: DRM device
1581 * @flags: startup flags
1582 *
1583 * The driver load routine has to do several things:
1584 * - drive output discovery via intel_modeset_init()
1585 * - initialize the memory manager
1586 * - allocate initial config memory
1587 * - setup the DRM framebuffer with the allocated memory
1588 */
42f5551d 1589int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
0673ad47
CW		 1590{
1591 struct drm_i915_private *dev_priv;
1592 int ret;
7d87a7f7 1593
a09d0ba1
CW		 1594 if (i915.nuclear_pageflip)
1595 driver.driver_features |= DRIVER_ATOMIC;
1596
0673ad47
CW		 1597 ret = -ENOMEM;
1598 dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
1599 if (dev_priv)
1600 ret = drm_dev_init(&dev_priv->drm, &driver, &pdev->dev);
1601 if (ret) {
1602 dev_printk(KERN_ERR, &pdev->dev,
1603 "[" DRM_NAME ":%s] allocation failed\n", __func__);
1604 kfree(dev_priv);
1605 return ret;
1606 }
72bbf0af 1607
0673ad47
CW		 1608 /* Must be set before calling __i915_printk */
1609 dev_priv->drm.pdev = pdev;
1610 dev_priv->drm.dev_private = dev_priv;
1611 dev_priv->dev = &dev_priv->drm;
719388e1 1612
0673ad47
CW		 1613 ret = pci_enable_device(pdev);
1614 if (ret)
1615 goto out_free_priv;
1347f5b4 1616
0673ad47 1617 pci_set_drvdata(pdev, &dev_priv->drm);
ef11bdb3 1618
0673ad47
CW		 1619 ret = i915_driver_init_early(dev_priv, ent);
1620 if (ret < 0)
1621 goto out_pci_disable;
ef11bdb3 1622
0673ad47 1623 intel_runtime_pm_get(dev_priv);
1da177e4 1624
0673ad47
CW		 1625 ret = i915_driver_init_mmio(dev_priv);
1626 if (ret < 0)
1627 goto out_runtime_pm_put;
79e53945 1628
0673ad47
CW		 1629 ret = i915_driver_init_hw(dev_priv);
1630 if (ret < 0)
1631 goto out_cleanup_mmio;
30c964a6
RB		 1632
1633 /*
0673ad47
CW		 1634 * TODO: move the vblank init and parts of modeset init steps into one
1635 * of the i915_driver_init_/i915_driver_register functions according
1636 * to the role/effect of the given init step.
30c964a6 1637 */
0673ad47
CW		 1638 if (INTEL_INFO(dev_priv)->num_pipes) {
1639 ret = drm_vblank_init(dev_priv->dev,
1640 INTEL_INFO(dev_priv)->num_pipes);
1641 if (ret)
1642 goto out_cleanup_hw;
30c964a6
RB		 1643 }
1644
0673ad47
CW		 1645 ret = i915_load_modeset_init(dev_priv->dev);
1646 if (ret < 0)
1647 goto out_cleanup_vblank;
1648
1649 i915_driver_register(dev_priv);
1650
1651 intel_runtime_pm_enable(dev_priv);
1652
1653 intel_runtime_pm_put(dev_priv);
1654
1655 return 0;
1656
1657out_cleanup_vblank:
1658 drm_vblank_cleanup(dev_priv->dev);
1659out_cleanup_hw:
1660 i915_driver_cleanup_hw(dev_priv);
1661out_cleanup_mmio:
1662 i915_driver_cleanup_mmio(dev_priv);
1663out_runtime_pm_put:
1664 intel_runtime_pm_put(dev_priv);
1665 i915_driver_cleanup_early(dev_priv);
1666out_pci_disable:
1667 pci_disable_device(pdev);
1668out_free_priv:
1669 i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret);
1670 drm_dev_unref(&dev_priv->drm);
30c964a6
RB		 1671 return ret;
1672}
1673
42f5551d 1674void i915_driver_unload(struct drm_device *dev)
3bad0781
ZW		 1675{
1676 struct drm_i915_private *dev_priv = dev->dev_private;
3bad0781 1677
0673ad47
CW		 1678 intel_fbdev_fini(dev);
1679
42f5551d
CW		 1680 if (i915_gem_suspend(dev))
1681 DRM_ERROR("failed to idle hardware; continuing to unload!\n");
ce1bb329 1682
0673ad47
CW		 1683 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
1684
1685 i915_driver_unregister(dev_priv);
1686
1687 drm_vblank_cleanup(dev);
1688
1689 intel_modeset_cleanup(dev);
1690
3bad0781 1691 /*
0673ad47
CW		 1692 * free the memory space allocated for the child device
1693 * config parsed from VBT
3bad0781 1694 */
0673ad47
CW		 1695 if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) {
1696 kfree(dev_priv->vbt.child_dev);
1697 dev_priv->vbt.child_dev = NULL;
1698 dev_priv->vbt.child_dev_num = 0;
1699 }
1700 kfree(dev_priv->vbt.sdvo_lvds_vbt_mode);
1701 dev_priv->vbt.sdvo_lvds_vbt_mode = NULL;
1702 kfree(dev_priv->vbt.lfp_lvds_vbt_mode);
1703 dev_priv->vbt.lfp_lvds_vbt_mode = NULL;
3bad0781 1704
0673ad47
CW		 1705 vga_switcheroo_unregister_client(dev->pdev);
1706 vga_client_register(dev->pdev, NULL, NULL, NULL);
bcdb72ac 1707
0673ad47 1708 intel_csr_ucode_fini(dev_priv);
bcdb72ac 1709
0673ad47
CW		 1710 /* Free error state after interrupts are fully disabled. */
1711 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
1712 i915_destroy_error_state(dev);
1713
1714 /* Flush any outstanding unpin_work. */
1715 flush_workqueue(dev_priv->wq);
1716
1717 intel_guc_fini(dev);
1718 i915_gem_fini(dev);
1719 intel_fbc_cleanup_cfb(dev_priv);
1720
1721 intel_power_domains_fini(dev_priv);
1722
1723 i915_driver_cleanup_hw(dev_priv);
1724 i915_driver_cleanup_mmio(dev_priv);
1725
1726 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
1727
1728 i915_driver_cleanup_early(dev_priv);
3bad0781
ZW		 1729}
1730
0673ad47 1731static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
2911a35b 1732{
0673ad47 1733 int ret;
2911a35b 1734
0673ad47
CW		 1735 ret = i915_gem_open(dev, file);
1736 if (ret)
1737 return ret;
2911a35b 1738
0673ad47
CW		 1739 return 0;
1740}
71386ef9 1741
0673ad47
CW		 1742/**
1743 * i915_driver_lastclose - clean up after all DRM clients have exited
1744 * @dev: DRM device
1745 *
1746 * Take care of cleaning up after all DRM clients have exited. In the
1747 * mode setting case, we want to restore the kernel's initial mode (just
1748 * in case the last client left us in a bad state).
1749 *
1750 * Additionally, in the non-mode setting case, we'll tear down the GTT
1751 * and DMA structures, since the kernel won't be using them, and clea
1752 * up any GEM state.
1753 */
1754static void i915_driver_lastclose(struct drm_device *dev)
1755{
1756 intel_fbdev_restore_mode(dev);
1757 vga_switcheroo_process_delayed_switch();
1758}
2911a35b 1759
0673ad47
CW		 1760static void i915_driver_preclose(struct drm_device *dev, struct drm_file *file)
1761{
1762 mutex_lock(&dev->struct_mutex);
1763 i915_gem_context_close(dev, file);
1764 i915_gem_release(dev, file);
1765 mutex_unlock(&dev->struct_mutex);
1766}
1767
1768static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
1769{
1770 struct drm_i915_file_private *file_priv = file->driver_priv;
1771
1772 kfree(file_priv);
2911a35b
BW		 1773}
1774
07f9cd0b
ID		 1775static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
1776{
1777 struct drm_device *dev = dev_priv->dev;
19c8054c 1778 struct intel_encoder *encoder;
07f9cd0b
ID		 1779
1780 drm_modeset_lock_all(dev);
19c8054c
JN		 1781 for_each_intel_encoder(dev, encoder)
1782 if (encoder->suspend)
1783 encoder->suspend(encoder);
07f9cd0b
ID		 1784 drm_modeset_unlock_all(dev);
1785}
1786
1a5df187
PZ		 1787static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
1788 bool rpm_resume);
507e126e 1789static int vlv_suspend_complete(struct drm_i915_private *dev_priv);
f75a1985 1790
bc87229f
ID		 1791static bool suspend_to_idle(struct drm_i915_private *dev_priv)
1792{
1793#if IS_ENABLED(CONFIG_ACPI_SLEEP)
1794 if (acpi_target_system_state() < ACPI_STATE_S3)
1795 return true;
1796#endif
1797 return false;
1798}
ebc32824 1799
5e365c39 1800static int i915_drm_suspend(struct drm_device *dev)
ba8bbcf6 1801{
61caf87c 1802 struct drm_i915_private *dev_priv = dev->dev_private;
e5747e3a 1803 pci_power_t opregion_target_state;
d5818938 1804 int error;
61caf87c 1805
b8efb17b
ZR		 1806 /* ignore lid events during suspend */
1807 mutex_lock(&dev_priv->modeset_restore_lock);
1808 dev_priv->modeset_restore = MODESET_SUSPENDED;
1809 mutex_unlock(&dev_priv->modeset_restore_lock);
1810
1f814dac
ID		 1811 disable_rpm_wakeref_asserts(dev_priv);
1812
c67a470b
PZ		 1813 /* We do a lot of poking in a lot of registers, make sure they work
1814 * properly. */
da7e29bd 1815 intel_display_set_init_power(dev_priv, true);
cb10799c 1816
5bcf719b
DA		 1817 drm_kms_helper_poll_disable(dev);
1818
ba8bbcf6 1819 pci_save_state(dev->pdev);
ba8bbcf6 1820
d5818938
DV		 1821 error = i915_gem_suspend(dev);
1822 if (error) {
1823 dev_err(&dev->pdev->dev,
1824 "GEM idle failed, resume might fail\n");
1f814dac 1825 goto out;
d5818938 1826 }
db1b76ca 1827
a1c41994
AD		 1828 intel_guc_suspend(dev);
1829
dc97997a 1830 intel_suspend_gt_powersave(dev_priv);
a261b246 1831
6b72d486 1832 intel_display_suspend(dev);
2eb5252e 1833
d5818938 1834 intel_dp_mst_suspend(dev);
7d708ee4 1835
d5818938
DV		 1836 intel_runtime_pm_disable_interrupts(dev_priv);
1837 intel_hpd_cancel_work(dev_priv);
09b64267 1838
d5818938 1839 intel_suspend_encoders(dev_priv);
0e32b39c 1840
d5818938 1841 intel_suspend_hw(dev);
5669fcac 1842
828c7908
BW		 1843 i915_gem_suspend_gtt_mappings(dev);
1844
9e06dd39
JB		 1845 i915_save_state(dev);
1846
bc87229f 1847 opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
6f9f4b7a 1848 intel_opregion_notify_adapter(dev_priv, opregion_target_state);
e5747e3a 1849
dc97997a 1850 intel_uncore_forcewake_reset(dev_priv, false);
03d92e47 1851 intel_opregion_unregister(dev_priv);
8ee1c3db 1852
82e3b8c1 1853 intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
3fa016a0 1854
62d5d69b
MK		 1855 dev_priv->suspend_count++;
1856
85e90679
KCA		 1857 intel_display_set_init_power(dev_priv, false);
1858
f74ed08d 1859 intel_csr_ucode_suspend(dev_priv);
f514c2d8 1860
1f814dac
ID		 1861out:
1862 enable_rpm_wakeref_asserts(dev_priv);
1863
1864 return error;
84b79f8d
RW		 1865}
1866
ab3be73f 1867static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation)
c3c09c95
ID		 1868{
1869 struct drm_i915_private *dev_priv = drm_dev->dev_private;
bc87229f 1870 bool fw_csr;
c3c09c95
ID		 1871 int ret;
1872
1f814dac
ID		 1873 disable_rpm_wakeref_asserts(dev_priv);
1874
a7c8125f
ID		 1875 fw_csr = !IS_BROXTON(dev_priv) &&
1876 suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload;
bc87229f
ID		 1877 /*
1878 * In case of firmware assisted context save/restore don't manually
1879 * deinit the power domains. This also means the CSR/DMC firmware will
1880 * stay active, it will power down any HW resources as required and
1881 * also enable deeper system power states that would be blocked if the
1882 * firmware was inactive.
1883 */
1884 if (!fw_csr)
1885 intel_power_domains_suspend(dev_priv);
73dfc227 1886
507e126e 1887 ret = 0;
b8aea3d1 1888 if (IS_BROXTON(dev_priv))
507e126e 1889 bxt_enable_dc9(dev_priv);
b8aea3d1 1890 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
507e126e
ID		 1891 hsw_enable_pc8(dev_priv);
1892 else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1893 ret = vlv_suspend_complete(dev_priv);
c3c09c95
ID		 1894
1895 if (ret) {
1896 DRM_ERROR("Suspend complete failed: %d\n", ret);
bc87229f
ID		 1897 if (!fw_csr)
1898 intel_power_domains_init_hw(dev_priv, true);
c3c09c95 1899
1f814dac 1900 goto out;
c3c09c95
ID		 1901 }
1902
1903 pci_disable_device(drm_dev->pdev);
ab3be73f 1904 /*
54875571 1905 * During hibernation on some platforms the BIOS may try to access
ab3be73f
ID		 1906 * the device even though it's already in D3 and hang the machine. So
1907 * leave the device in D0 on those platforms and hope the BIOS will
54875571
ID		 1908 * power down the device properly. The issue was seen on multiple old
1909 * GENs with different BIOS vendors, so having an explicit blacklist
1910 * is inpractical; apply the workaround on everything pre GEN6. The
1911 * platforms where the issue was seen:
1912 * Lenovo Thinkpad X301, X61s, X60, T60, X41
1913 * Fujitsu FSC S7110
1914 * Acer Aspire 1830T
ab3be73f 1915 */
54875571 1916 if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6))
ab3be73f 1917 pci_set_power_state(drm_dev->pdev, PCI_D3hot);
c3c09c95 1918
bc87229f
ID		 1919 dev_priv->suspended_to_idle = suspend_to_idle(dev_priv);
1920
1f814dac
ID		 1921out:
1922 enable_rpm_wakeref_asserts(dev_priv);
1923
1924 return ret;
c3c09c95
ID		 1925}
1926
1751fcf9 1927int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state)
84b79f8d
RW		 1928{
1929 int error;
1930
1931 if (!dev || !dev->dev_private) {
1932 DRM_ERROR("dev: %p\n", dev);
1933 DRM_ERROR("DRM not initialized, aborting suspend.\n");
1934 return -ENODEV;
1935 }
1936
0b14cbd2
ID		 1937 if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND &&
1938 state.event != PM_EVENT_FREEZE))
1939 return -EINVAL;
5bcf719b
DA		 1940
1941 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1942 return 0;
6eecba33 1943
5e365c39 1944 error = i915_drm_suspend(dev);
84b79f8d
RW		 1945 if (error)
1946 return error;
1947
ab3be73f 1948 return i915_drm_suspend_late(dev, false);
ba8bbcf6
JB		 1949}
1950
5e365c39 1951static int i915_drm_resume(struct drm_device *dev)
76c4b250
ID		 1952{
1953 struct drm_i915_private *dev_priv = dev->dev_private;
ac840ae5 1954 int ret;
9d49c0ef 1955
1f814dac
ID		 1956 disable_rpm_wakeref_asserts(dev_priv);
1957
ac840ae5
VS		 1958 ret = i915_ggtt_enable_hw(dev);
1959 if (ret)
1960 DRM_ERROR("failed to re-enable GGTT\n");
1961
f74ed08d
ID		 1962 intel_csr_ucode_resume(dev_priv);
1963
d5818938
DV		 1964 mutex_lock(&dev->struct_mutex);
1965 i915_gem_restore_gtt_mappings(dev);
1966 mutex_unlock(&dev->struct_mutex);
9d49c0ef 1967
61caf87c 1968 i915_restore_state(dev);
6f9f4b7a 1969 intel_opregion_setup(dev_priv);
61caf87c 1970
d5818938
DV		 1971 intel_init_pch_refclk(dev);
1972 drm_mode_config_reset(dev);
1833b134 1973
364aece0
PA		 1974 /*
1975 * Interrupts have to be enabled before any batches are run. If not the
1976 * GPU will hang. i915_gem_init_hw() will initiate batches to
1977 * update/restore the context.
1978 *
1979 * Modeset enabling in intel_modeset_init_hw() also needs working
1980 * interrupts.
1981 */
1982 intel_runtime_pm_enable_interrupts(dev_priv);
1983
d5818938
DV		 1984 mutex_lock(&dev->struct_mutex);
1985 if (i915_gem_init_hw(dev)) {
1986 DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
805de8f4 1987 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
d5818938
DV		 1988 }
1989 mutex_unlock(&dev->struct_mutex);
226485e9 1990
a1c41994
AD		 1991 intel_guc_resume(dev);
1992
d5818938 1993 intel_modeset_init_hw(dev);
24576d23 1994
d5818938
DV		 1995 spin_lock_irq(&dev_priv->irq_lock);
1996 if (dev_priv->display.hpd_irq_setup)
91d14251 1997 dev_priv->display.hpd_irq_setup(dev_priv);
d5818938 1998 spin_unlock_irq(&dev_priv->irq_lock);
0e32b39c 1999
d5818938 2000 intel_dp_mst_resume(dev);
e7d6f7d7 2001
a16b7658
L		 2002 intel_display_resume(dev);
2003
d5818938
DV		 2004 /*
2005 * ... but also need to make sure that hotplug processing
2006 * doesn't cause havoc. Like in the driver load code we don't
2007 * bother with the tiny race here where we might loose hotplug
2008 * notifications.
2009 * */
2010 intel_hpd_init(dev_priv);
2011 /* Config may have changed between suspend and resume */
2012 drm_helper_hpd_irq_event(dev);
1daed3fb 2013
03d92e47 2014 intel_opregion_register(dev_priv);
44834a67 2015
82e3b8c1 2016 intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
073f34d9 2017
b8efb17b
ZR		 2018 mutex_lock(&dev_priv->modeset_restore_lock);
2019 dev_priv->modeset_restore = MODESET_DONE;
2020 mutex_unlock(&dev_priv->modeset_restore_lock);
8a187455 2021
6f9f4b7a 2022 intel_opregion_notify_adapter(dev_priv, PCI_D0);
e5747e3a 2023
ee6f280e
ID		 2024 drm_kms_helper_poll_enable(dev);
2025
1f814dac
ID		 2026 enable_rpm_wakeref_asserts(dev_priv);
2027
074c6ada 2028 return 0;
84b79f8d
RW		 2029}
2030
5e365c39 2031static int i915_drm_resume_early(struct drm_device *dev)
84b79f8d 2032{
36d61e67 2033 struct drm_i915_private *dev_priv = dev->dev_private;
44410cd0 2034 int ret;
36d61e67 2035
76c4b250
ID		 2036 /*
2037 * We have a resume ordering issue with the snd-hda driver also
2038 * requiring our device to be power up. Due to the lack of a
2039 * parent/child relationship we currently solve this with an early
2040 * resume hook.
2041 *
2042 * FIXME: This should be solved with a special hdmi sink device or
2043 * similar so that power domains can be employed.
2044 */
44410cd0
ID		 2045
2046 /*
2047 * Note that we need to set the power state explicitly, since we
2048 * powered off the device during freeze and the PCI core won't power
2049 * it back up for us during thaw. Powering off the device during
2050 * freeze is not a hard requirement though, and during the
2051 * suspend/resume phases the PCI core makes sure we get here with the
2052 * device powered on. So in case we change our freeze logic and keep
2053 * the device powered we can also remove the following set power state
2054 * call.
2055 */
2056 ret = pci_set_power_state(dev->pdev, PCI_D0);
2057 if (ret) {
2058 DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret);
2059 goto out;
2060 }
2061
2062 /*
2063 * Note that pci_enable_device() first enables any parent bridge
2064 * device and only then sets the power state for this device. The
2065 * bridge enabling is a nop though, since bridge devices are resumed
2066 * first. The order of enabling power and enabling the device is
2067 * imposed by the PCI core as described above, so here we preserve the
2068 * same order for the freeze/thaw phases.
2069 *
2070 * TODO: eventually we should remove pci_disable_device() /
2071 * pci_enable_enable_device() from suspend/resume. Due to how they
2072 * depend on the device enable refcount we can't anyway depend on them
2073 * disabling/enabling the device.
2074 */
bc87229f
ID		 2075 if (pci_enable_device(dev->pdev)) {
2076 ret = -EIO;
2077 goto out;
2078 }
84b79f8d
RW		 2079
2080 pci_set_master(dev->pdev);
2081
1f814dac
ID		 2082 disable_rpm_wakeref_asserts(dev_priv);
2083
666a4537 2084 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1a5df187 2085 ret = vlv_resume_prepare(dev_priv, false);
36d61e67 2086 if (ret)
ff0b187f
DL		 2087 DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
2088 ret);
36d61e67 2089
dc97997a 2090 intel_uncore_early_sanitize(dev_priv, true);
efee833a 2091
dc97997a 2092 if (IS_BROXTON(dev_priv)) {
da2f41d1
ID		 2093 if (!dev_priv->suspended_to_idle)
2094 gen9_sanitize_dc_state(dev_priv);
507e126e 2095 bxt_disable_dc9(dev_priv);
da2f41d1 2096 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
a9a6b73a 2097 hsw_disable_pc8(dev_priv);
da2f41d1 2098 }
efee833a 2099
dc97997a 2100 intel_uncore_sanitize(dev_priv);
bc87229f 2101
a7c8125f
ID		 2102 if (IS_BROXTON(dev_priv) ||
2103 !(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload))
bc87229f
ID		 2104 intel_power_domains_init_hw(dev_priv, true);
2105
6e35e8ab
ID		 2106 enable_rpm_wakeref_asserts(dev_priv);
2107
bc87229f
ID		 2108out:
2109 dev_priv->suspended_to_idle = false;
36d61e67
ID		 2110
2111 return ret;
76c4b250
ID		 2112}
2113
1751fcf9 2114int i915_resume_switcheroo(struct drm_device *dev)
76c4b250 2115{
50a0072f 2116 int ret;
76c4b250 2117
097dd837
ID		 2118 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2119 return 0;
2120
5e365c39 2121 ret = i915_drm_resume_early(dev);
50a0072f
ID		 2122 if (ret)
2123 return ret;
2124
5a17514e
ID		 2125 return i915_drm_resume(dev);
2126}
2127
11ed50ec 2128/**
f3953dcb 2129 * i915_reset - reset chip after a hang
11ed50ec 2130 * @dev: drm device to reset
11ed50ec
BG		 2131 *
2132 * Reset the chip. Useful if a hang is detected. Returns zero on successful
2133 * reset or otherwise an error code.
2134 *
2135 * Procedure is fairly simple:
2136 * - reset the chip using the reset reg
2137 * - re-init context state
2138 * - re-init hardware status page
2139 * - re-init ring buffer
2140 * - re-init interrupt state
2141 * - re-init display
2142 */
c033666a 2143int i915_reset(struct drm_i915_private *dev_priv)
11ed50ec 2144{
c033666a 2145 struct drm_device *dev = dev_priv->dev;
d98c52cf
CW		 2146 struct i915_gpu_error *error = &dev_priv->gpu_error;
2147 unsigned reset_counter;
0573ed4a 2148 int ret;
11ed50ec 2149
dc97997a 2150 intel_reset_gt_powersave(dev_priv);
dbea3cea 2151
d54a02c0 2152 mutex_lock(&dev->struct_mutex);
11ed50ec 2153
d98c52cf
CW		 2154 /* Clear any previous failed attempts at recovery. Time to try again. */
2155 atomic_andnot(I915_WEDGED, &error->reset_counter);
77f01230 2156
d98c52cf
CW		 2157 /* Clear the reset-in-progress flag and increment the reset epoch. */
2158 reset_counter = atomic_inc_return(&error->reset_counter);
2159 if (WARN_ON(__i915_reset_in_progress(reset_counter))) {
2160 ret = -EIO;
2161 goto error;
2162 }
2163
2164 i915_gem_reset(dev);
2e7c8ee7 2165
dc97997a 2166 ret = intel_gpu_reset(dev_priv, ALL_ENGINES);
be62acb4
MK		 2167
2168 /* Also reset the gpu hangman. */
d98c52cf 2169 if (error->stop_rings != 0) {
be62acb4 2170 DRM_INFO("Simulated gpu hang, resetting stop_rings\n");
d98c52cf 2171 error->stop_rings = 0;
be62acb4 2172 if (ret == -ENODEV) {
f2d91a2c
DV		 2173 DRM_INFO("Reset not implemented, but ignoring "
2174 "error for simulated gpu hangs\n");
be62acb4
MK		 2175 ret = 0;
2176 }
2e7c8ee7 2177 }
be62acb4 2178
d8f2716a
DV		 2179 if (i915_stop_ring_allow_warn(dev_priv))
2180 pr_notice("drm/i915: Resetting chip after gpu hang\n");
2181
0573ed4a 2182 if (ret) {
804e59a8
CW		 2183 if (ret != -ENODEV)
2184 DRM_ERROR("Failed to reset chip: %i\n", ret);
2185 else
2186 DRM_DEBUG_DRIVER("GPU reset disabled\n");
d98c52cf 2187 goto error;
11ed50ec
BG		 2188 }
2189
1362b776
VS		 2190 intel_overlay_reset(dev_priv);
2191
11ed50ec
BG		 2192 /* Ok, now get things going again... */
2193
2194 /*
2195 * Everything depends on having the GTT running, so we need to start
2196 * there. Fortunately we don't need to do this unless we reset the
2197 * chip at a PCI level.
2198 *
2199 * Next we need to restore the context, but we don't use those
2200 * yet either...
2201 *
2202 * Ring buffer needs to be re-initialized in the KMS case, or if X
2203 * was running at the time of the reset (i.e. we weren't VT
2204 * switched away).
2205 */
33d30a9c 2206 ret = i915_gem_init_hw(dev);
33d30a9c
DV		 2207 if (ret) {
2208 DRM_ERROR("Failed hw init on reset %d\n", ret);
d98c52cf 2209 goto error;
11ed50ec
BG		 2210 }
2211
d98c52cf
CW		 2212 mutex_unlock(&dev->struct_mutex);
2213
33d30a9c
DV		 2214 /*
2215 * rps/rc6 re-init is necessary to restore state lost after the
2216 * reset and the re-install of gt irqs. Skip for ironlake per
2217 * previous concerns that it doesn't respond well to some forms
2218 * of re-init after reset.
2219 */
2220 if (INTEL_INFO(dev)->gen > 5)
dc97997a 2221 intel_enable_gt_powersave(dev_priv);
33d30a9c 2222
11ed50ec 2223 return 0;
d98c52cf
CW		 2224
2225error:
2226 atomic_or(I915_WEDGED, &error->reset_counter);
2227 mutex_unlock(&dev->struct_mutex);
2228 return ret;
11ed50ec
BG		 2229}
2230
84b79f8d 2231static int i915_pm_suspend(struct device *dev)
112b715e 2232{
84b79f8d
RW		 2233 struct pci_dev *pdev = to_pci_dev(dev);
2234 struct drm_device *drm_dev = pci_get_drvdata(pdev);
112b715e 2235
84b79f8d
RW		 2236 if (!drm_dev || !drm_dev->dev_private) {
2237 dev_err(dev, "DRM not initialized, aborting suspend.\n");
2238 return -ENODEV;
2239 }
112b715e 2240
5bcf719b
DA		 2241 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2242 return 0;
2243
5e365c39 2244 return i915_drm_suspend(drm_dev);
76c4b250
ID		 2245}
2246
2247static int i915_pm_suspend_late(struct device *dev)
2248{
888d0d42 2249 struct drm_device *drm_dev = dev_to_i915(dev)->dev;
76c4b250
ID		 2250
2251 /*
c965d995 2252 * We have a suspend ordering issue with the snd-hda driver also
76c4b250
ID		 2253 * requiring our device to be power up. Due to the lack of a
2254 * parent/child relationship we currently solve this with an late
2255 * suspend hook.
2256 *
2257 * FIXME: This should be solved with a special hdmi sink device or
2258 * similar so that power domains can be employed.
2259 */
2260 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2261 return 0;
112b715e 2262
ab3be73f
ID		 2263 return i915_drm_suspend_late(drm_dev, false);
2264}
2265
2266static int i915_pm_poweroff_late(struct device *dev)
2267{
2268 struct drm_device *drm_dev = dev_to_i915(dev)->dev;
2269
2270 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2271 return 0;
2272
2273 return i915_drm_suspend_late(drm_dev, true);
cbda12d7
ZW		 2274}
2275
76c4b250
ID		 2276static int i915_pm_resume_early(struct device *dev)
2277{
888d0d42 2278 struct drm_device *drm_dev = dev_to_i915(dev)->dev;
76c4b250 2279
097dd837
ID		 2280 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2281 return 0;
2282
5e365c39 2283 return i915_drm_resume_early(drm_dev);
76c4b250
ID		 2284}
2285
84b79f8d 2286static int i915_pm_resume(struct device *dev)
cbda12d7 2287{
888d0d42 2288 struct drm_device *drm_dev = dev_to_i915(dev)->dev;
84b79f8d 2289
097dd837
ID		 2290 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2291 return 0;
2292
5a17514e 2293 return i915_drm_resume(drm_dev);
cbda12d7
ZW		 2294}
2295
1f19ac2a
CW		 2296/* freeze: before creating the hibernation_image */
2297static int i915_pm_freeze(struct device *dev)
2298{
2299 return i915_pm_suspend(dev);
2300}
2301
2302static int i915_pm_freeze_late(struct device *dev)
2303{
461fb99c
CW		 2304 int ret;
2305
2306 ret = i915_pm_suspend_late(dev);
2307 if (ret)
2308 return ret;
2309
2310 ret = i915_gem_freeze_late(dev_to_i915(dev));
2311 if (ret)
2312 return ret;
2313
2314 return 0;
1f19ac2a
CW		 2315}
2316
2317/* thaw: called after creating the hibernation image, but before turning off. */
2318static int i915_pm_thaw_early(struct device *dev)
2319{
2320 return i915_pm_resume_early(dev);
2321}
2322
2323static int i915_pm_thaw(struct device *dev)
2324{
2325 return i915_pm_resume(dev);
2326}
2327
2328/* restore: called after loading the hibernation image. */
2329static int i915_pm_restore_early(struct device *dev)
2330{
2331 return i915_pm_resume_early(dev);
2332}
2333
2334static int i915_pm_restore(struct device *dev)
2335{
2336 return i915_pm_resume(dev);
2337}
2338
ddeea5b0
ID		 2339/*
2340 * Save all Gunit registers that may be lost after a D3 and a subsequent
2341 * S0i[R123] transition. The list of registers needing a save/restore is
2342 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
2343 * registers in the following way:
2344 * - Driver: saved/restored by the driver
2345 * - Punit : saved/restored by the Punit firmware
2346 * - No, w/o marking: no need to save/restore, since the register is R/O or
2347 * used internally by the HW in a way that doesn't depend
2348 * keeping the content across a suspend/resume.
2349 * - Debug : used for debugging
2350 *
2351 * We save/restore all registers marked with 'Driver', with the following
2352 * exceptions:
2353 * - Registers out of use, including also registers marked with 'Debug'.
2354 * These have no effect on the driver's operation, so we don't save/restore
2355 * them to reduce the overhead.
2356 * - Registers that are fully setup by an initialization function called from
2357 * the resume path. For example many clock gating and RPS/RC6 registers.
2358 * - Registers that provide the right functionality with their reset defaults.
2359 *
2360 * TODO: Except for registers that based on the above 3 criteria can be safely
2361 * ignored, we save/restore all others, practically treating the HW context as
2362 * a black-box for the driver. Further investigation is needed to reduce the
2363 * saved/restored registers even further, by following the same 3 criteria.
2364 */
2365static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv)
2366{
2367 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
2368 int i;
2369
2370 /* GAM 0x4000-0x4770 */
2371 s->wr_watermark = I915_READ(GEN7_WR_WATERMARK);
2372 s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL);
2373 s->arb_mode = I915_READ(ARB_MODE);
2374 s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0);
2375 s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1);
2376
2377 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
22dfe79f 2378 s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
ddeea5b0
ID		 2379
2380 s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
b5f1c97f 2381 s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT);
ddeea5b0
ID		 2382
2383 s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7);
2384 s->ecochk = I915_READ(GAM_ECOCHK);
2385 s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7);
2386 s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7);
2387
2388 s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR);
2389
2390 /* MBC 0x9024-0x91D0, 0x8500 */
2391 s->g3dctl = I915_READ(VLV_G3DCTL);
2392 s->gsckgctl = I915_READ(VLV_GSCKGCTL);
2393 s->mbctl = I915_READ(GEN6_MBCTL);
2394
2395 /* GCP 0x9400-0x9424, 0x8100-0x810C */
2396 s->ucgctl1 = I915_READ(GEN6_UCGCTL1);
2397 s->ucgctl3 = I915_READ(GEN6_UCGCTL3);
2398 s->rcgctl1 = I915_READ(GEN6_RCGCTL1);
2399 s->rcgctl2 = I915_READ(GEN6_RCGCTL2);
2400 s->rstctl = I915_READ(GEN6_RSTCTL);
2401 s->misccpctl = I915_READ(GEN7_MISCCPCTL);
2402
2403 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
2404 s->gfxpause = I915_READ(GEN6_GFXPAUSE);
2405 s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC);
2406 s->rpdeuc = I915_READ(GEN6_RPDEUC);
2407 s->ecobus = I915_READ(ECOBUS);
2408 s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL);
2409 s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT);
2410 s->rp_deucsw = I915_READ(GEN6_RPDEUCSW);
2411 s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR);
2412 s->rcedata = I915_READ(VLV_RCEDATA);
2413 s->spare2gh = I915_READ(VLV_SPAREG2H);
2414
2415 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
2416 s->gt_imr = I915_READ(GTIMR);
2417 s->gt_ier = I915_READ(GTIER);
2418 s->pm_imr = I915_READ(GEN6_PMIMR);
2419 s->pm_ier = I915_READ(GEN6_PMIER);
2420
2421 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
22dfe79f 2422 s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
ddeea5b0
ID		 2423
2424 /* GT SA CZ domain, 0x100000-0x138124 */
2425 s->tilectl = I915_READ(TILECTL);
2426 s->gt_fifoctl = I915_READ(GTFIFOCTL);
2427 s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL);
2428 s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2429 s->pmwgicz = I915_READ(VLV_PMWGICZ);
2430
2431 /* Gunit-Display CZ domain, 0x182028-0x1821CF */
2432 s->gu_ctl0 = I915_READ(VLV_GU_CTL0);
2433 s->gu_ctl1 = I915_READ(VLV_GU_CTL1);
9c25210f 2434 s->pcbr = I915_READ(VLV_PCBR);
ddeea5b0
ID		 2435 s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2);
2436
2437 /*
2438 * Not saving any of:
2439 * DFT, 0x9800-0x9EC0
2440 * SARB, 0xB000-0xB1FC
2441 * GAC, 0x5208-0x524C, 0x14000-0x14C000
2442 * PCI CFG
2443 */
2444}
2445
2446static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv)
2447{
2448 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
2449 u32 val;
2450 int i;
2451
2452 /* GAM 0x4000-0x4770 */
2453 I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark);
2454 I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl);
2455 I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16));
2456 I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0);
2457 I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1);
2458
2459 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
22dfe79f 2460 I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
ddeea5b0
ID		 2461
2462 I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
b5f1c97f 2463 I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
ddeea5b0
ID		 2464
2465 I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp);
2466 I915_WRITE(GAM_ECOCHK, s->ecochk);
2467 I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp);
2468 I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp);
2469
2470 I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr);
2471
2472 /* MBC 0x9024-0x91D0, 0x8500 */
2473 I915_WRITE(VLV_G3DCTL, s->g3dctl);
2474 I915_WRITE(VLV_GSCKGCTL, s->gsckgctl);
2475 I915_WRITE(GEN6_MBCTL, s->mbctl);
2476
2477 /* GCP 0x9400-0x9424, 0x8100-0x810C */
2478 I915_WRITE(GEN6_UCGCTL1, s->ucgctl1);
2479 I915_WRITE(GEN6_UCGCTL3, s->ucgctl3);
2480 I915_WRITE(GEN6_RCGCTL1, s->rcgctl1);
2481 I915_WRITE(GEN6_RCGCTL2, s->rcgctl2);
2482 I915_WRITE(GEN6_RSTCTL, s->rstctl);
2483 I915_WRITE(GEN7_MISCCPCTL, s->misccpctl);
2484
2485 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
2486 I915_WRITE(GEN6_GFXPAUSE, s->gfxpause);
2487 I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc);
2488 I915_WRITE(GEN6_RPDEUC, s->rpdeuc);
2489 I915_WRITE(ECOBUS, s->ecobus);
2490 I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl);
2491 I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout);
2492 I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw);
2493 I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr);
2494 I915_WRITE(VLV_RCEDATA, s->rcedata);
2495 I915_WRITE(VLV_SPAREG2H, s->spare2gh);
2496
2497 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
2498 I915_WRITE(GTIMR, s->gt_imr);
2499 I915_WRITE(GTIER, s->gt_ier);
2500 I915_WRITE(GEN6_PMIMR, s->pm_imr);
2501 I915_WRITE(GEN6_PMIER, s->pm_ier);
2502
2503 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
22dfe79f 2504 I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
ddeea5b0
ID		 2505
2506 /* GT SA CZ domain, 0x100000-0x138124 */
2507 I915_WRITE(TILECTL, s->tilectl);
2508 I915_WRITE(GTFIFOCTL, s->gt_fifoctl);
2509 /*
2510 * Preserve the GT allow wake and GFX force clock bit, they are not
2511 * be restored, as they are used to control the s0ix suspend/resume
2512 * sequence by the caller.
2513 */
2514 val = I915_READ(VLV_GTLC_WAKE_CTRL);
2515 val &= VLV_GTLC_ALLOWWAKEREQ;
2516 val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
2517 I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2518
2519 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2520 val &= VLV_GFX_CLK_FORCE_ON_BIT;
2521 val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
2522 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2523
2524 I915_WRITE(VLV_PMWGICZ, s->pmwgicz);
2525
2526 /* Gunit-Display CZ domain, 0x182028-0x1821CF */
2527 I915_WRITE(VLV_GU_CTL0, s->gu_ctl0);
2528 I915_WRITE(VLV_GU_CTL1, s->gu_ctl1);
9c25210f 2529 I915_WRITE(VLV_PCBR, s->pcbr);
ddeea5b0
ID		 2530 I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2);
2531}
2532
650ad970
ID		 2533int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
2534{
2535 u32 val;
2536 int err;
2537
650ad970 2538#define COND (I915_READ(VLV_GTLC_SURVIVABILITY_REG) & VLV_GFX_CLK_STATUS_BIT)
650ad970
ID		 2539
2540 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2541 val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
2542 if (force_on)
2543 val |= VLV_GFX_CLK_FORCE_ON_BIT;
2544 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2545
2546 if (!force_on)
2547 return 0;
2548
8d4eee9c 2549 err = wait_for(COND, 20);
650ad970
ID		 2550 if (err)
2551 DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n",
2552 I915_READ(VLV_GTLC_SURVIVABILITY_REG));
2553
2554 return err;
2555#undef COND
2556}
2557
ddeea5b0
ID		 2558static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
2559{
2560 u32 val;
2561 int err = 0;
2562
2563 val = I915_READ(VLV_GTLC_WAKE_CTRL);
2564 val &= ~VLV_GTLC_ALLOWWAKEREQ;
2565 if (allow)
2566 val |= VLV_GTLC_ALLOWWAKEREQ;
2567 I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2568 POSTING_READ(VLV_GTLC_WAKE_CTRL);
2569
2570#define COND (!!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEACK) == \
2571 allow)
2572 err = wait_for(COND, 1);
2573 if (err)
2574 DRM_ERROR("timeout disabling GT waking\n");
2575 return err;
2576#undef COND
2577}
2578
2579static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
2580 bool wait_for_on)
2581{
2582 u32 mask;
2583 u32 val;
2584 int err;
2585
2586 mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
2587 val = wait_for_on ? mask : 0;
2588#define COND ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val)
2589 if (COND)
2590 return 0;
2591
2592 DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n",
87ad3212
JN		 2593 onoff(wait_for_on),
2594 I915_READ(VLV_GTLC_PW_STATUS));
ddeea5b0
ID		 2595
2596 /*
2597 * RC6 transitioning can be delayed up to 2 msec (see
2598 * valleyview_enable_rps), use 3 msec for safety.
2599 */
2600 err = wait_for(COND, 3);
2601 if (err)
2602 DRM_ERROR("timeout waiting for GT wells to go %s\n",
87ad3212 2603 onoff(wait_for_on));
ddeea5b0
ID		 2604
2605 return err;
2606#undef COND
2607}
2608
2609static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
2610{
2611 if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
2612 return;
2613
6fa283b0 2614 DRM_DEBUG_DRIVER("GT register access while GT waking disabled\n");
ddeea5b0
ID		 2615 I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
2616}
2617
ebc32824 2618static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
ddeea5b0
ID		 2619{
2620 u32 mask;
2621 int err;
2622
2623 /*
2624 * Bspec defines the following GT well on flags as debug only, so
2625 * don't treat them as hard failures.
2626 */
2627 (void)vlv_wait_for_gt_wells(dev_priv, false);
2628
2629 mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
2630 WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask);
2631
2632 vlv_check_no_gt_access(dev_priv);
2633
2634 err = vlv_force_gfx_clock(dev_priv, true);
2635 if (err)
2636 goto err1;
2637
2638 err = vlv_allow_gt_wake(dev_priv, false);
2639 if (err)
2640 goto err2;
98711167 2641
2d1fe073 2642 if (!IS_CHERRYVIEW(dev_priv))
98711167 2643 vlv_save_gunit_s0ix_state(dev_priv);
ddeea5b0
ID		 2644
2645 err = vlv_force_gfx_clock(dev_priv, false);
2646 if (err)
2647 goto err2;
2648
2649 return 0;
2650
2651err2:
2652 /* For safety always re-enable waking and disable gfx clock forcing */
2653 vlv_allow_gt_wake(dev_priv, true);
2654err1:
2655 vlv_force_gfx_clock(dev_priv, false);
2656
2657 return err;
2658}
2659
016970be
SK		 2660static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
2661 bool rpm_resume)
ddeea5b0
ID		 2662{
2663 struct drm_device *dev = dev_priv->dev;
2664 int err;
2665 int ret;
2666
2667 /*
2668 * If any of the steps fail just try to continue, that's the best we
2669 * can do at this point. Return the first error code (which will also
2670 * leave RPM permanently disabled).
2671 */
2672 ret = vlv_force_gfx_clock(dev_priv, true);
2673
2d1fe073 2674 if (!IS_CHERRYVIEW(dev_priv))
98711167 2675 vlv_restore_gunit_s0ix_state(dev_priv);
ddeea5b0
ID		 2676
2677 err = vlv_allow_gt_wake(dev_priv, true);
2678 if (!ret)
2679 ret = err;
2680
2681 err = vlv_force_gfx_clock(dev_priv, false);
2682 if (!ret)
2683 ret = err;
2684
2685 vlv_check_no_gt_access(dev_priv);
2686
016970be
SK		 2687 if (rpm_resume) {
2688 intel_init_clock_gating(dev);
2689 i915_gem_restore_fences(dev);
2690 }
ddeea5b0
ID		 2691
2692 return ret;
2693}
2694
97bea207 2695static int intel_runtime_suspend(struct device *device)
8a187455
PZ		 2696{
2697 struct pci_dev *pdev = to_pci_dev(device);
2698 struct drm_device *dev = pci_get_drvdata(pdev);
2699 struct drm_i915_private *dev_priv = dev->dev_private;
0ab9cfeb 2700 int ret;
8a187455 2701
dc97997a 2702 if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6())))
c6df39b5
ID		 2703 return -ENODEV;
2704
604effb7
ID		 2705 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
2706 return -ENODEV;
2707
8a187455
PZ		 2708 DRM_DEBUG_KMS("Suspending device\n");
2709
d6102977
ID		 2710 /*
2711 * We could deadlock here in case another thread holding struct_mutex
2712 * calls RPM suspend concurrently, since the RPM suspend will wait
2713 * first for this RPM suspend to finish. In this case the concurrent
2714 * RPM resume will be followed by its RPM suspend counterpart. Still
2715 * for consistency return -EAGAIN, which will reschedule this suspend.
2716 */
2717 if (!mutex_trylock(&dev->struct_mutex)) {
2718 DRM_DEBUG_KMS("device lock contention, deffering suspend\n");
2719 /*
2720 * Bump the expiration timestamp, otherwise the suspend won't
2721 * be rescheduled.
2722 */
2723 pm_runtime_mark_last_busy(device);
2724
2725 return -EAGAIN;
2726 }
1f814dac
ID		 2727
2728 disable_rpm_wakeref_asserts(dev_priv);
2729
d6102977
ID		 2730 /*
2731 * We are safe here against re-faults, since the fault handler takes
2732 * an RPM reference.
2733 */
2734 i915_gem_release_all_mmaps(dev_priv);
2735 mutex_unlock(&dev->struct_mutex);
2736
825f2728
JL		 2737 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
2738
a1c41994
AD		 2739 intel_guc_suspend(dev);
2740
dc97997a 2741 intel_suspend_gt_powersave(dev_priv);
2eb5252e 2742 intel_runtime_pm_disable_interrupts(dev_priv);
b5478bcd 2743
507e126e
ID		 2744 ret = 0;
2745 if (IS_BROXTON(dev_priv)) {
2746 bxt_display_core_uninit(dev_priv);
2747 bxt_enable_dc9(dev_priv);
2748 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2749 hsw_enable_pc8(dev_priv);
2750 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
2751 ret = vlv_suspend_complete(dev_priv);
2752 }
2753
0ab9cfeb
ID		 2754 if (ret) {
2755 DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
b963291c 2756 intel_runtime_pm_enable_interrupts(dev_priv);
0ab9cfeb 2757
1f814dac
ID		 2758 enable_rpm_wakeref_asserts(dev_priv);
2759
0ab9cfeb
ID		 2760 return ret;
2761 }
a8a8bd54 2762
dc97997a 2763 intel_uncore_forcewake_reset(dev_priv, false);
1f814dac
ID		 2764
2765 enable_rpm_wakeref_asserts(dev_priv);
2766 WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
55ec45c2 2767
bc3b9346 2768 if (intel_uncore_arm_unclaimed_mmio_detection(dev_priv))
55ec45c2
MK		 2769 DRM_ERROR("Unclaimed access detected prior to suspending\n");
2770
8a187455 2771 dev_priv->pm.suspended = true;
1fb2362b
KCA		 2772
2773 /*
c8a0bd42
PZ		 2774 * FIXME: We really should find a document that references the arguments
2775 * used below!
1fb2362b 2776 */
6f9f4b7a 2777 if (IS_BROADWELL(dev_priv)) {
d37ae19a
PZ		 2778 /*
2779 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
2780 * being detected, and the call we do at intel_runtime_resume()
2781 * won't be able to restore them. Since PCI_D3hot matches the
2782 * actual specification and appears to be working, use it.
2783 */
6f9f4b7a 2784 intel_opregion_notify_adapter(dev_priv, PCI_D3hot);
d37ae19a 2785 } else {
c8a0bd42
PZ		 2786 /*
2787 * current versions of firmware which depend on this opregion
2788 * notification have repurposed the D1 definition to mean
2789 * "runtime suspended" vs. what you would normally expect (D3)
2790 * to distinguish it from notifications that might be sent via
2791 * the suspend path.
2792 */
6f9f4b7a 2793 intel_opregion_notify_adapter(dev_priv, PCI_D1);
c8a0bd42 2794 }
8a187455 2795
59bad947 2796 assert_forcewakes_inactive(dev_priv);
dc9fb09c 2797
a8a8bd54 2798 DRM_DEBUG_KMS("Device suspended\n");
8a187455
PZ		 2799 return 0;
2800}
2801
97bea207 2802static int intel_runtime_resume(struct device *device)
8a187455
PZ		 2803{
2804 struct pci_dev *pdev = to_pci_dev(device);
2805 struct drm_device *dev = pci_get_drvdata(pdev);
2806 struct drm_i915_private *dev_priv = dev->dev_private;
1a5df187 2807 int ret = 0;
8a187455 2808
604effb7
ID		 2809 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
2810 return -ENODEV;
8a187455
PZ		 2811
2812 DRM_DEBUG_KMS("Resuming device\n");
2813
1f814dac
ID		 2814 WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
2815 disable_rpm_wakeref_asserts(dev_priv);
2816
6f9f4b7a 2817 intel_opregion_notify_adapter(dev_priv, PCI_D0);
8a187455 2818 dev_priv->pm.suspended = false;
55ec45c2
MK		 2819 if (intel_uncore_unclaimed_mmio(dev_priv))
2820 DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n");
8a187455 2821
a1c41994
AD		 2822 intel_guc_resume(dev);
2823
1a5df187
PZ		 2824 if (IS_GEN6(dev_priv))
2825 intel_init_pch_refclk(dev);
31335cec 2826
507e126e
ID		 2827 if (IS_BROXTON(dev)) {
2828 bxt_disable_dc9(dev_priv);
2829 bxt_display_core_init(dev_priv, true);
f62c79b3
ID		 2830 if (dev_priv->csr.dmc_payload &&
2831 (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
2832 gen9_enable_dc5(dev_priv);
507e126e 2833 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
1a5df187 2834 hsw_disable_pc8(dev_priv);
507e126e 2835 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1a5df187 2836 ret = vlv_resume_prepare(dev_priv, true);
507e126e 2837 }
1a5df187 2838
0ab9cfeb
ID		 2839 /*
2840 * No point of rolling back things in case of an error, as the best
2841 * we can do is to hope that things will still work (and disable RPM).
2842 */
92b806d3 2843 i915_gem_init_swizzling(dev);
dc97997a 2844 gen6_update_ring_freq(dev_priv);
92b806d3 2845
b963291c 2846 intel_runtime_pm_enable_interrupts(dev_priv);
08d8a232
VS		 2847
2848 /*
2849 * On VLV/CHV display interrupts are part of the display
2850 * power well, so hpd is reinitialized from there. For
2851 * everyone else do it here.
2852 */
666a4537 2853 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
08d8a232
VS		 2854 intel_hpd_init(dev_priv);
2855
dc97997a 2856 intel_enable_gt_powersave(dev_priv);
b5478bcd 2857
1f814dac
ID		 2858 enable_rpm_wakeref_asserts(dev_priv);
2859
0ab9cfeb
ID		 2860 if (ret)
2861 DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret);
2862 else
2863 DRM_DEBUG_KMS("Device resumed\n");
2864
2865 return ret;
8a187455
PZ		 2866}
2867
42f5551d 2868const struct dev_pm_ops i915_pm_ops = {
5545dbbf
ID		 2869 /*
2870 * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
2871 * PMSG_RESUME]
2872 */
0206e353 2873 .suspend = i915_pm_suspend,
76c4b250
ID		 2874 .suspend_late = i915_pm_suspend_late,
2875 .resume_early = i915_pm_resume_early,
0206e353 2876 .resume = i915_pm_resume,
5545dbbf
ID		 2877
2878 /*
2879 * S4 event handlers
2880 * @freeze, @freeze_late : called (1) before creating the
2881 * hibernation image [PMSG_FREEZE] and
2882 * (2) after rebooting, before restoring
2883 * the image [PMSG_QUIESCE]
2884 * @thaw, @thaw_early : called (1) after creating the hibernation
2885 * image, before writing it [PMSG_THAW]
2886 * and (2) after failing to create or
2887 * restore the image [PMSG_RECOVER]
2888 * @poweroff, @poweroff_late: called after writing the hibernation
2889 * image, before rebooting [PMSG_HIBERNATE]
2890 * @restore, @restore_early : called after rebooting and restoring the
2891 * hibernation image [PMSG_RESTORE]
2892 */
1f19ac2a
CW		 2893 .freeze = i915_pm_freeze,
2894 .freeze_late = i915_pm_freeze_late,
2895 .thaw_early = i915_pm_thaw_early,
2896 .thaw = i915_pm_thaw,
36d61e67 2897 .poweroff = i915_pm_suspend,
ab3be73f 2898 .poweroff_late = i915_pm_poweroff_late,
1f19ac2a
CW		 2899 .restore_early = i915_pm_restore_early,
2900 .restore = i915_pm_restore,
5545dbbf
ID		 2901
2902 /* S0ix (via runtime suspend) event handlers */
97bea207
PZ		 2903 .runtime_suspend = intel_runtime_suspend,
2904 .runtime_resume = intel_runtime_resume,
cbda12d7
ZW		 2905};
2906
78b68556 2907static const struct vm_operations_struct i915_gem_vm_ops = {
de151cf6 2908 .fault = i915_gem_fault,
ab00b3e5
JB		 2909 .open = drm_gem_vm_open,
2910 .close = drm_gem_vm_close,
de151cf6
JB		 2911};
2912
e08e96de
AV		 2913static const struct file_operations i915_driver_fops = {
2914 .owner = THIS_MODULE,
2915 .open = drm_open,
2916 .release = drm_release,
2917 .unlocked_ioctl = drm_ioctl,
2918 .mmap = drm_gem_mmap,
2919 .poll = drm_poll,
e08e96de
AV		 2920 .read = drm_read,
2921#ifdef CONFIG_COMPAT
2922 .compat_ioctl = i915_compat_ioctl,
2923#endif
2924 .llseek = noop_llseek,
2925};
2926
0673ad47
CW		 2927static int
2928i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data,
2929 struct drm_file *file)
2930{
2931 return -ENODEV;
2932}
2933
2934static const struct drm_ioctl_desc i915_ioctls[] = {
2935 DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2936 DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH),
2937 DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH),
2938 DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH),
2939 DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
2940 DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
2941 DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
2942 DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2943 DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
2944 DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
2945 DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2946 DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH),
2947 DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2948 DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2949 DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH),
2950 DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
2951 DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2952 DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2953 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH),
2954 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW),
2955 DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2956 DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2957 DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2958 DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
2959 DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
2960 DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2961 DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2962 DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2963 DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
2964 DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
2965 DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
2966 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
2967 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW),
2968 DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
2969 DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
2970 DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW),
2971 DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW),
2972 DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
2973 DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0),
2974 DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
2975 DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
2976 DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
2977 DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW),
2978 DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW),
2979 DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2980 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
2981 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
2982 DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
2983 DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW),
2984 DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
2985 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
2986 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
2987};
2988
1da177e4 2989static struct drm_driver driver = {
0c54781b
MW		 2990 /* Don't use MTRRs here; the Xserver or userspace app should
2991 * deal with them for Intel hardware.
792d2b9a 2992 */
673a394b 2993 .driver_features =
10ba5012 2994 DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME |
1751fcf9 2995 DRIVER_RENDER | DRIVER_MODESET,
673a394b 2996 .open = i915_driver_open,
22eae947
DA		 2997 .lastclose = i915_driver_lastclose,
2998 .preclose = i915_driver_preclose,
673a394b 2999 .postclose = i915_driver_postclose,
915b4d11 3000 .set_busid = drm_pci_set_busid,
d8e29209 3001
673a394b 3002 .gem_free_object = i915_gem_free_object,
de151cf6 3003 .gem_vm_ops = &i915_gem_vm_ops,
1286ff73
DV		 3004
3005 .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
3006 .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
3007 .gem_prime_export = i915_gem_prime_export,
3008 .gem_prime_import = i915_gem_prime_import,
3009
ff72145b 3010 .dumb_create = i915_gem_dumb_create,
da6b51d0 3011 .dumb_map_offset = i915_gem_mmap_gtt,
43387b37 3012 .dumb_destroy = drm_gem_dumb_destroy,
1da177e4 3013 .ioctls = i915_ioctls,
0673ad47 3014 .num_ioctls = ARRAY_SIZE(i915_ioctls),
e08e96de 3015 .fops = &i915_driver_fops,
22eae947
DA		 3016 .name = DRIVER_NAME,
3017 .desc = DRIVER_DESC,
3018 .date = DRIVER_DATE,
3019 .major = DRIVER_MAJOR,
3020 .minor = DRIVER_MINOR,
3021 .patchlevel = DRIVER_PATCHLEVEL,
1da177e4 3022};
ubuntu-bionic-kernel mirror