]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/gpu/drm/i915/i915_drv.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Restore current RPS state after reset
[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / i915 / i915_drv.c
CommitLineData
1da177e4
LT		 1/* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
2 */
0d6aa60b 3/*
bc54fd1a 4 *
1da177e4
LT		 5 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
6 * All Rights Reserved.
bc54fd1a
DA		 7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
23 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
0d6aa60b 28 */
1da177e4 29
e5747e3a 30#include <linux/acpi.h>
0673ad47
CW		 31#include <linux/device.h>
32#include <linux/oom.h>
e0cd3608 33#include <linux/module.h>
0673ad47
CW		 34#include <linux/pci.h>
35#include <linux/pm.h>
d6102977 36#include <linux/pm_runtime.h>
0673ad47
CW		 37#include <linux/pnp.h>
38#include <linux/slab.h>
39#include <linux/vgaarb.h>
704ab614 40#include <linux/vga_switcheroo.h>
0673ad47
CW		 41#include <linux/vt.h>
42#include <acpi/video.h>
43
44#include <drm/drmP.h>
760285e7 45#include <drm/drm_crtc_helper.h>
0673ad47
CW		 46#include <drm/i915_drm.h>
47
48#include "i915_drv.h"
49#include "i915_trace.h"
50#include "i915_vgpu.h"
51#include "intel_drv.h"
79e53945 52
112b715e
KH		 53static struct drm_driver driver;
54
0673ad47
CW		 55static unsigned int i915_load_fail_count;
56
57bool __i915_inject_load_failure(const char *func, int line)
58{
59 if (i915_load_fail_count >= i915.inject_load_failure)
60 return false;
61
62 if (++i915_load_fail_count == i915.inject_load_failure) {
63 DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n",
64 i915.inject_load_failure, func, line);
65 return true;
66 }
67
68 return false;
69}
70
71#define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI"
72#define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \
73 "providing the dmesg log by booting with drm.debug=0xf"
74
75void
76__i915_printk(struct drm_i915_private *dev_priv, const char *level,
77 const char *fmt, ...)
78{
79 static bool shown_bug_once;
c49d13ee 80 struct device *kdev = dev_priv->drm.dev;
0673ad47
CW		 81 bool is_error = level[1] <= KERN_ERR[1];
82 bool is_debug = level[1] == KERN_DEBUG[1];
83 struct va_format vaf;
84 va_list args;
85
86 if (is_debug && !(drm_debug & DRM_UT_DRIVER))
87 return;
88
89 va_start(args, fmt);
90
91 vaf.fmt = fmt;
92 vaf.va = &args;
93
c49d13ee 94 dev_printk(level, kdev, "[" DRM_NAME ":%ps] %pV",
0673ad47
CW		 95 __builtin_return_address(0), &vaf);
96
97 if (is_error && !shown_bug_once) {
c49d13ee 98 dev_notice(kdev, "%s", FDO_BUG_MSG);
0673ad47
CW		 99 shown_bug_once = true;
100 }
101
102 va_end(args);
103}
104
105static bool i915_error_injected(struct drm_i915_private *dev_priv)
106{
107 return i915.inject_load_failure &&
108 i915_load_fail_count == i915.inject_load_failure;
109}
110
111#define i915_load_error(dev_priv, fmt, ...) \
112 __i915_printk(dev_priv, \
113 i915_error_injected(dev_priv) ? KERN_DEBUG : KERN_ERR, \
114 fmt, ##__VA_ARGS__)
115
116
117static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
118{
119 enum intel_pch ret = PCH_NOP;
120
121 /*
122 * In a virtualized passthrough environment we can be in a
123 * setup where the ISA bridge is not able to be passed through.
124 * In this case, a south bridge can be emulated and we have to
125 * make an educated guess as to which PCH is really there.
126 */
127
128 if (IS_GEN5(dev)) {
129 ret = PCH_IBX;
130 DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n");
131 } else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
132 ret = PCH_CPT;
133 DRM_DEBUG_KMS("Assuming CouarPoint PCH\n");
134 } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
135 ret = PCH_LPT;
136 DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
137 } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
138 ret = PCH_SPT;
139 DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
140 }
141
142 return ret;
143}
144
145static void intel_detect_pch(struct drm_device *dev)
146{
fac5e23e 147 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 148 struct pci_dev *pch = NULL;
149
150 /* In all current cases, num_pipes is equivalent to the PCH_NOP setting
151 * (which really amounts to a PCH but no South Display).
152 */
153 if (INTEL_INFO(dev)->num_pipes == 0) {
154 dev_priv->pch_type = PCH_NOP;
155 return;
156 }
157
158 /*
159 * The reason to probe ISA bridge instead of Dev31:Fun0 is to
160 * make graphics device passthrough work easy for VMM, that only
161 * need to expose ISA bridge to let driver know the real hardware
162 * underneath. This is a requirement from virtualization team.
163 *
164 * In some virtualized environments (e.g. XEN), there is irrelevant
165 * ISA bridge in the system. To work reliably, we should scan trhough
166 * all the ISA bridge devices and check for the first match, instead
167 * of only checking the first one.
168 */
169 while ((pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, pch))) {
170 if (pch->vendor == PCI_VENDOR_ID_INTEL) {
171 unsigned short id = pch->device & INTEL_PCH_DEVICE_ID_MASK;
172 dev_priv->pch_id = id;
173
174 if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
175 dev_priv->pch_type = PCH_IBX;
176 DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
177 WARN_ON(!IS_GEN5(dev));
178 } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
179 dev_priv->pch_type = PCH_CPT;
180 DRM_DEBUG_KMS("Found CougarPoint PCH\n");
181 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
182 } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
183 /* PantherPoint is CPT compatible */
184 dev_priv->pch_type = PCH_CPT;
185 DRM_DEBUG_KMS("Found PantherPoint PCH\n");
186 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
187 } else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
188 dev_priv->pch_type = PCH_LPT;
189 DRM_DEBUG_KMS("Found LynxPoint PCH\n");
190 WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
191 WARN_ON(IS_HSW_ULT(dev) || IS_BDW_ULT(dev));
192 } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
193 dev_priv->pch_type = PCH_LPT;
194 DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
195 WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
196 WARN_ON(!IS_HSW_ULT(dev) && !IS_BDW_ULT(dev));
197 } else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
198 dev_priv->pch_type = PCH_SPT;
199 DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
200 WARN_ON(!IS_SKYLAKE(dev) &&
201 !IS_KABYLAKE(dev));
202 } else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
203 dev_priv->pch_type = PCH_SPT;
204 DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
205 WARN_ON(!IS_SKYLAKE(dev) &&
206 !IS_KABYLAKE(dev));
22dea0be
RV		 207 } else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) {
208 dev_priv->pch_type = PCH_KBP;
209 DRM_DEBUG_KMS("Found KabyPoint PCH\n");
210 WARN_ON(!IS_KABYLAKE(dev));
0673ad47
CW		 211 } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
212 (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) ||
213 ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) &&
214 pch->subsystem_vendor ==
215 PCI_SUBVENDOR_ID_REDHAT_QUMRANET &&
216 pch->subsystem_device ==
217 PCI_SUBDEVICE_ID_QEMU)) {
218 dev_priv->pch_type = intel_virt_detect_pch(dev);
219 } else
220 continue;
221
222 break;
223 }
224 }
225 if (!pch)
226 DRM_DEBUG_KMS("No PCH found.\n");
227
228 pci_dev_put(pch);
229}
230
0673ad47
CW		 231static int i915_getparam(struct drm_device *dev, void *data,
232 struct drm_file *file_priv)
233{
fac5e23e 234 struct drm_i915_private *dev_priv = to_i915(dev);
52a05c30 235 struct pci_dev *pdev = dev_priv->drm.pdev;
0673ad47
CW		 236 drm_i915_getparam_t *param = data;
237 int value;
238
239 switch (param->param) {
240 case I915_PARAM_IRQ_ACTIVE:
241 case I915_PARAM_ALLOW_BATCHBUFFER:
242 case I915_PARAM_LAST_DISPATCH:
243 /* Reject all old ums/dri params. */
244 return -ENODEV;
245 case I915_PARAM_CHIPSET_ID:
52a05c30 246 value = pdev->device;
0673ad47
CW		 247 break;
248 case I915_PARAM_REVISION:
52a05c30 249 value = pdev->revision;
0673ad47 250 break;
0673ad47
CW		 251 case I915_PARAM_NUM_FENCES_AVAIL:
252 value = dev_priv->num_fence_regs;
253 break;
254 case I915_PARAM_HAS_OVERLAY:
255 value = dev_priv->overlay ? 1 : 0;
256 break;
0673ad47
CW		 257 case I915_PARAM_HAS_BSD:
258 value = intel_engine_initialized(&dev_priv->engine[VCS]);
259 break;
260 case I915_PARAM_HAS_BLT:
261 value = intel_engine_initialized(&dev_priv->engine[BCS]);
262 break;
263 case I915_PARAM_HAS_VEBOX:
264 value = intel_engine_initialized(&dev_priv->engine[VECS]);
265 break;
266 case I915_PARAM_HAS_BSD2:
267 value = intel_engine_initialized(&dev_priv->engine[VCS2]);
268 break;
0673ad47 269 case I915_PARAM_HAS_EXEC_CONSTANTS:
16162470 270 value = INTEL_GEN(dev_priv) >= 4;
0673ad47
CW		 271 break;
272 case I915_PARAM_HAS_LLC:
16162470 273 value = HAS_LLC(dev_priv);
0673ad47
CW		 274 break;
275 case I915_PARAM_HAS_WT:
16162470 276 value = HAS_WT(dev_priv);
0673ad47
CW		 277 break;
278 case I915_PARAM_HAS_ALIASING_PPGTT:
16162470 279 value = USES_PPGTT(dev_priv);
0673ad47
CW		 280 break;
281 case I915_PARAM_HAS_SEMAPHORES:
39df9190 282 value = i915.semaphores;
0673ad47 283 break;
0673ad47
CW		 284 case I915_PARAM_HAS_SECURE_BATCHES:
285 value = capable(CAP_SYS_ADMIN);
286 break;
0673ad47
CW		 287 case I915_PARAM_CMD_PARSER_VERSION:
288 value = i915_cmd_parser_get_version(dev_priv);
289 break;
0673ad47 290 case I915_PARAM_SUBSLICE_TOTAL:
57ec171e 291 value = sseu_subslice_total(&INTEL_INFO(dev_priv)->sseu);
0673ad47
CW		 292 if (!value)
293 return -ENODEV;
294 break;
295 case I915_PARAM_EU_TOTAL:
43b67998 296 value = INTEL_INFO(dev_priv)->sseu.eu_total;
0673ad47
CW		 297 if (!value)
298 return -ENODEV;
299 break;
300 case I915_PARAM_HAS_GPU_RESET:
301 value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv);
302 break;
303 case I915_PARAM_HAS_RESOURCE_STREAMER:
16162470 304 value = HAS_RESOURCE_STREAMER(dev_priv);
0673ad47 305 break;
37f501af 306 case I915_PARAM_HAS_POOLED_EU:
16162470 307 value = HAS_POOLED_EU(dev_priv);
37f501af 308 break;
309 case I915_PARAM_MIN_EU_IN_POOL:
43b67998 310 value = INTEL_INFO(dev_priv)->sseu.min_eu_in_pool;
37f501af 311 break;
4cc69075
CW		 312 case I915_PARAM_MMAP_GTT_VERSION:
313 /* Though we've started our numbering from 1, and so class all
314 * earlier versions as 0, in effect their value is undefined as
315 * the ioctl will report EINVAL for the unknown param!
316 */
317 value = i915_gem_mmap_gtt_version();
318 break;
16162470
DW		 319 case I915_PARAM_MMAP_VERSION:
320 /* Remember to bump this if the version changes! */
321 case I915_PARAM_HAS_GEM:
322 case I915_PARAM_HAS_PAGEFLIPPING:
323 case I915_PARAM_HAS_EXECBUF2: /* depends on GEM */
324 case I915_PARAM_HAS_RELAXED_FENCING:
325 case I915_PARAM_HAS_COHERENT_RINGS:
326 case I915_PARAM_HAS_RELAXED_DELTA:
327 case I915_PARAM_HAS_GEN7_SOL_RESET:
328 case I915_PARAM_HAS_WAIT_TIMEOUT:
329 case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
330 case I915_PARAM_HAS_PINNED_BATCHES:
331 case I915_PARAM_HAS_EXEC_NO_RELOC:
332 case I915_PARAM_HAS_EXEC_HANDLE_LUT:
333 case I915_PARAM_HAS_COHERENT_PHYS_GTT:
334 case I915_PARAM_HAS_EXEC_SOFTPIN:
335 /* For the time being all of these are always true;
336 * if some supported hardware does not have one of these
337 * features this value needs to be provided from
338 * INTEL_INFO(), a feature macro, or similar.
339 */
340 value = 1;
341 break;
0673ad47
CW		 342 default:
343 DRM_DEBUG("Unknown parameter %d\n", param->param);
344 return -EINVAL;
345 }
346
dda33009 347 if (put_user(value, param->value))
0673ad47 348 return -EFAULT;
0673ad47
CW		 349
350 return 0;
351}
352
353static int i915_get_bridge_dev(struct drm_device *dev)
354{
fac5e23e 355 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 356
357 dev_priv->bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
358 if (!dev_priv->bridge_dev) {
359 DRM_ERROR("bridge device not found\n");
360 return -1;
361 }
362 return 0;
363}
364
365/* Allocate space for the MCH regs if needed, return nonzero on error */
366static int
367intel_alloc_mchbar_resource(struct drm_device *dev)
368{
fac5e23e 369 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 370 int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
371 u32 temp_lo, temp_hi = 0;
372 u64 mchbar_addr;
373 int ret;
374
375 if (INTEL_INFO(dev)->gen >= 4)
376 pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
377 pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
378 mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
379
380 /* If ACPI doesn't have it, assume we need to allocate it ourselves */
381#ifdef CONFIG_PNP
382 if (mchbar_addr &&
383 pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
384 return 0;
385#endif
386
387 /* Get some space for it */
388 dev_priv->mch_res.name = "i915 MCHBAR";
389 dev_priv->mch_res.flags = IORESOURCE_MEM;
390 ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,
391 &dev_priv->mch_res,
392 MCHBAR_SIZE, MCHBAR_SIZE,
393 PCIBIOS_MIN_MEM,
394 0, pcibios_align_resource,
395 dev_priv->bridge_dev);
396 if (ret) {
397 DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret);
398 dev_priv->mch_res.start = 0;
399 return ret;
400 }
401
402 if (INTEL_INFO(dev)->gen >= 4)
403 pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
404 upper_32_bits(dev_priv->mch_res.start));
405
406 pci_write_config_dword(dev_priv->bridge_dev, reg,
407 lower_32_bits(dev_priv->mch_res.start));
408 return 0;
409}
410
411/* Setup MCHBAR if possible, return true if we should disable it again */
412static void
413intel_setup_mchbar(struct drm_device *dev)
414{
fac5e23e 415 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 416 int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
417 u32 temp;
418 bool enabled;
419
420 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
421 return;
422
423 dev_priv->mchbar_need_disable = false;
424
425 if (IS_I915G(dev) || IS_I915GM(dev)) {
426 pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);
427 enabled = !!(temp & DEVEN_MCHBAR_EN);
428 } else {
429 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
430 enabled = temp & 1;
431 }
432
433 /* If it's already enabled, don't have to do anything */
434 if (enabled)
435 return;
436
437 if (intel_alloc_mchbar_resource(dev))
438 return;
439
440 dev_priv->mchbar_need_disable = true;
441
442 /* Space is allocated or reserved, so enable it. */
443 if (IS_I915G(dev) || IS_I915GM(dev)) {
444 pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
445 temp | DEVEN_MCHBAR_EN);
446 } else {
447 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
448 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);
449 }
450}
451
452static void
453intel_teardown_mchbar(struct drm_device *dev)
454{
fac5e23e 455 struct drm_i915_private *dev_priv = to_i915(dev);
0673ad47
CW		 456 int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
457
458 if (dev_priv->mchbar_need_disable) {
459 if (IS_I915G(dev) || IS_I915GM(dev)) {
460 u32 deven_val;
461
462 pci_read_config_dword(dev_priv->bridge_dev, DEVEN,
463 &deven_val);
464 deven_val &= ~DEVEN_MCHBAR_EN;
465 pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
466 deven_val);
467 } else {
468 u32 mchbar_val;
469
470 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg,
471 &mchbar_val);
472 mchbar_val &= ~1;
473 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg,
474 mchbar_val);
475 }
476 }
477
478 if (dev_priv->mch_res.start)
479 release_resource(&dev_priv->mch_res);
480}
481
482/* true = enable decode, false = disable decoder */
483static unsigned int i915_vga_set_decode(void *cookie, bool state)
484{
485 struct drm_device *dev = cookie;
486
487 intel_modeset_vga_set_state(dev, state);
488 if (state)
489 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
490 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
491 else
492 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
493}
494
495static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
496{
497 struct drm_device *dev = pci_get_drvdata(pdev);
498 pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
499
500 if (state == VGA_SWITCHEROO_ON) {
501 pr_info("switched on\n");
502 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
503 /* i915 resume handler doesn't set to D0 */
52a05c30 504 pci_set_power_state(pdev, PCI_D0);
0673ad47
CW		 505 i915_resume_switcheroo(dev);
506 dev->switch_power_state = DRM_SWITCH_POWER_ON;
507 } else {
508 pr_info("switched off\n");
509 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
510 i915_suspend_switcheroo(dev, pmm);
511 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
512 }
513}
514
515static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
516{
517 struct drm_device *dev = pci_get_drvdata(pdev);
518
519 /*
520 * FIXME: open_count is protected by drm_global_mutex but that would lead to
521 * locking inversion with the driver load path. And the access here is
522 * completely racy anyway. So don't bother with locking for now.
523 */
524 return dev->open_count == 0;
525}
526
527static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
528 .set_gpu_state = i915_switcheroo_set_state,
529 .reprobe = NULL,
530 .can_switch = i915_switcheroo_can_switch,
531};
532
533static void i915_gem_fini(struct drm_device *dev)
534{
535 struct drm_i915_private *dev_priv = to_i915(dev);
536
537 /*
538 * Neither the BIOS, ourselves or any other kernel
539 * expects the system to be in execlists mode on startup,
540 * so we need to reset the GPU back to legacy mode. And the only
541 * known way to disable logical contexts is through a GPU reset.
542 *
543 * So in order to leave the system in a known default configuration,
544 * always reset the GPU upon unload. Afterwards we then clean up the
545 * GEM state tracking, flushing off the requests and leaving the
546 * system in a known idle state.
547 *
548 * Note that is of the upmost importance that the GPU is idle and
549 * all stray writes are flushed *before* we dismantle the backing
550 * storage for the pinned objects.
551 *
552 * However, since we are uncertain that reseting the GPU on older
553 * machines is a good idea, we don't - just in case it leaves the
554 * machine in an unusable condition.
555 */
556 if (HAS_HW_CONTEXTS(dev)) {
557 int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
558 WARN_ON(reset && reset != -ENODEV);
559 }
560
561 mutex_lock(&dev->struct_mutex);
0673ad47
CW		 562 i915_gem_cleanup_engines(dev);
563 i915_gem_context_fini(dev);
564 mutex_unlock(&dev->struct_mutex);
565
566 WARN_ON(!list_empty(&to_i915(dev)->context_list));
567}
568
569static int i915_load_modeset_init(struct drm_device *dev)
570{
fac5e23e 571 struct drm_i915_private *dev_priv = to_i915(dev);
52a05c30 572 struct pci_dev *pdev = dev_priv->drm.pdev;
0673ad47
CW		 573 int ret;
574
575 if (i915_inject_load_failure())
576 return -ENODEV;
577
578 ret = intel_bios_init(dev_priv);
579 if (ret)
580 DRM_INFO("failed to find VBIOS tables\n");
581
582 /* If we have > 1 VGA cards, then we need to arbitrate access
583 * to the common VGA resources.
584 *
585 * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
586 * then we do not take part in VGA arbitration and the
587 * vga_client_register() fails with -ENODEV.
588 */
52a05c30 589 ret = vga_client_register(pdev, dev, NULL, i915_vga_set_decode);
0673ad47
CW		 590 if (ret && ret != -ENODEV)
591 goto out;
592
593 intel_register_dsm_handler();
594
52a05c30 595 ret = vga_switcheroo_register_client(pdev, &i915_switcheroo_ops, false);
0673ad47
CW		 596 if (ret)
597 goto cleanup_vga_client;
598
599 /* must happen before intel_power_domains_init_hw() on VLV/CHV */
600 intel_update_rawclk(dev_priv);
601
602 intel_power_domains_init_hw(dev_priv, false);
603
604 intel_csr_ucode_init(dev_priv);
605
606 ret = intel_irq_install(dev_priv);
607 if (ret)
608 goto cleanup_csr;
609
610 intel_setup_gmbus(dev);
611
612 /* Important: The output setup functions called by modeset_init need
613 * working irqs for e.g. gmbus and dp aux transfers. */
614 intel_modeset_init(dev);
615
616 intel_guc_init(dev);
617
618 ret = i915_gem_init(dev);
619 if (ret)
620 goto cleanup_irq;
621
622 intel_modeset_gem_init(dev);
623
624 if (INTEL_INFO(dev)->num_pipes == 0)
625 return 0;
626
627 ret = intel_fbdev_init(dev);
628 if (ret)
629 goto cleanup_gem;
630
631 /* Only enable hotplug handling once the fbdev is fully set up. */
632 intel_hpd_init(dev_priv);
633
634 drm_kms_helper_poll_init(dev);
635
636 return 0;
637
638cleanup_gem:
639 i915_gem_fini(dev);
640cleanup_irq:
641 intel_guc_fini(dev);
642 drm_irq_uninstall(dev);
643 intel_teardown_gmbus(dev);
644cleanup_csr:
645 intel_csr_ucode_fini(dev_priv);
646 intel_power_domains_fini(dev_priv);
52a05c30 647 vga_switcheroo_unregister_client(pdev);
0673ad47 648cleanup_vga_client:
52a05c30 649 vga_client_register(pdev, NULL, NULL, NULL);
0673ad47
CW		 650out:
651 return ret;
652}
653
654#if IS_ENABLED(CONFIG_FB)
655static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
656{
657 struct apertures_struct *ap;
91c8a326 658 struct pci_dev *pdev = dev_priv->drm.pdev;
0673ad47
CW		 659 struct i915_ggtt *ggtt = &dev_priv->ggtt;
660 bool primary;
661 int ret;
662
663 ap = alloc_apertures(1);
664 if (!ap)
665 return -ENOMEM;
666
667 ap->ranges[0].base = ggtt->mappable_base;
668 ap->ranges[0].size = ggtt->mappable_end;
669
670 primary =
671 pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
672
44adece5 673 ret = drm_fb_helper_remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
0673ad47
CW		 674
675 kfree(ap);
676
677 return ret;
678}
679#else
680static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
681{
682 return 0;
683}
684#endif
685
686#if !defined(CONFIG_VGA_CONSOLE)
687static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
688{
689 return 0;
690}
691#elif !defined(CONFIG_DUMMY_CONSOLE)
692static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
693{
694 return -ENODEV;
695}
696#else
697static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
698{
699 int ret = 0;
700
701 DRM_INFO("Replacing VGA console driver\n");
702
703 console_lock();
704 if (con_is_bound(&vga_con))
705 ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1);
706 if (ret == 0) {
707 ret = do_unregister_con_driver(&vga_con);
708
709 /* Ignore "already unregistered". */
710 if (ret == -ENODEV)
711 ret = 0;
712 }
713 console_unlock();
714
715 return ret;
716}
717#endif
718
0673ad47
CW		 719static void intel_init_dpio(struct drm_i915_private *dev_priv)
720{
721 /*
722 * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
723 * CHV x1 PHY (DP/HDMI D)
724 * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
725 */
726 if (IS_CHERRYVIEW(dev_priv)) {
727 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
728 DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
729 } else if (IS_VALLEYVIEW(dev_priv)) {
730 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
731 }
732}
733
734static int i915_workqueues_init(struct drm_i915_private *dev_priv)
735{
736 /*
737 * The i915 workqueue is primarily used for batched retirement of
738 * requests (and thus managing bo) once the task has been completed
739 * by the GPU. i915_gem_retire_requests() is called directly when we
740 * need high-priority retirement, such as waiting for an explicit
741 * bo.
742 *
743 * It is also used for periodic low-priority events, such as
744 * idle-timers and recording error state.
745 *
746 * All tasks on the workqueue are expected to acquire the dev mutex
747 * so there is no point in running more than one instance of the
748 * workqueue at any time. Use an ordered one.
749 */
750 dev_priv->wq = alloc_ordered_workqueue("i915", 0);
751 if (dev_priv->wq == NULL)
752 goto out_err;
753
754 dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
755 if (dev_priv->hotplug.dp_wq == NULL)
756 goto out_free_wq;
757
0673ad47
CW		 758 return 0;
759
0673ad47
CW		 760out_free_wq:
761 destroy_workqueue(dev_priv->wq);
762out_err:
763 DRM_ERROR("Failed to allocate workqueues.\n");
764
765 return -ENOMEM;
766}
767
768static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
769{
0673ad47
CW		 770 destroy_workqueue(dev_priv->hotplug.dp_wq);
771 destroy_workqueue(dev_priv->wq);
772}
773
774/**
775 * i915_driver_init_early - setup state not requiring device access
776 * @dev_priv: device private
777 *
778 * Initialize everything that is a "SW-only" state, that is state not
779 * requiring accessing the device or exposing the driver via kernel internal
780 * or userspace interfaces. Example steps belonging here: lock initialization,
781 * system memory allocation, setting up device specific attributes and
782 * function hooks not requiring accessing the device.
783 */
784static int i915_driver_init_early(struct drm_i915_private *dev_priv,
785 const struct pci_device_id *ent)
786{
787 const struct intel_device_info *match_info =
788 (struct intel_device_info *)ent->driver_data;
789 struct intel_device_info *device_info;
790 int ret = 0;
791
792 if (i915_inject_load_failure())
793 return -ENODEV;
794
795 /* Setup the write-once "constant" device info */
94b4f3ba 796 device_info = mkwrite_device_info(dev_priv);
0673ad47
CW		 797 memcpy(device_info, match_info, sizeof(*device_info));
798 device_info->device_id = dev_priv->drm.pdev->device;
799
800 BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE);
801 device_info->gen_mask = BIT(device_info->gen - 1);
802
803 spin_lock_init(&dev_priv->irq_lock);
804 spin_lock_init(&dev_priv->gpu_error.lock);
805 mutex_init(&dev_priv->backlight_lock);
806 spin_lock_init(&dev_priv->uncore.lock);
807 spin_lock_init(&dev_priv->mm.object_stat_lock);
808 spin_lock_init(&dev_priv->mmio_flip_lock);
809 mutex_init(&dev_priv->sb_lock);
810 mutex_init(&dev_priv->modeset_restore_lock);
811 mutex_init(&dev_priv->av_mutex);
812 mutex_init(&dev_priv->wm.wm_mutex);
813 mutex_init(&dev_priv->pps_mutex);
814
0b1de5d5
CW		 815 i915_memcpy_init_early(dev_priv);
816
0673ad47
CW		 817 ret = i915_workqueues_init(dev_priv);
818 if (ret < 0)
819 return ret;
820
821 ret = intel_gvt_init(dev_priv);
822 if (ret < 0)
823 goto err_workqueues;
824
825 /* This must be called before any calls to HAS_PCH_* */
826 intel_detect_pch(&dev_priv->drm);
827
828 intel_pm_setup(&dev_priv->drm);
829 intel_init_dpio(dev_priv);
830 intel_power_domains_init(dev_priv);
831 intel_irq_init(dev_priv);
832 intel_init_display_hooks(dev_priv);
833 intel_init_clock_gating_hooks(dev_priv);
834 intel_init_audio_hooks(dev_priv);
835 i915_gem_load_init(&dev_priv->drm);
836
36cdd013 837 intel_display_crc_init(dev_priv);
0673ad47 838
94b4f3ba 839 intel_device_info_dump(dev_priv);
0673ad47
CW		 840
841 /* Not all pre-production machines fall into this category, only the
842 * very first ones. Almost everything should work, except for maybe
843 * suspend/resume. And we don't implement workarounds that affect only
844 * pre-production machines. */
845 if (IS_HSW_EARLY_SDV(dev_priv))
846 DRM_INFO("This is an early pre-production Haswell machine. "
847 "It may not be fully functional.\n");
848
849 return 0;
850
851err_workqueues:
852 i915_workqueues_cleanup(dev_priv);
853 return ret;
854}
855
856/**
857 * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early()
858 * @dev_priv: device private
859 */
860static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
861{
91c8a326 862 i915_gem_load_cleanup(&dev_priv->drm);
0673ad47
CW		 863 i915_workqueues_cleanup(dev_priv);
864}
865
866static int i915_mmio_setup(struct drm_device *dev)
867{
868 struct drm_i915_private *dev_priv = to_i915(dev);
52a05c30 869 struct pci_dev *pdev = dev_priv->drm.pdev;
0673ad47
CW		 870 int mmio_bar;
871 int mmio_size;
872
873 mmio_bar = IS_GEN2(dev) ? 1 : 0;
874 /*
875 * Before gen4, the registers and the GTT are behind different BARs.
876 * However, from gen4 onwards, the registers and the GTT are shared
877 * in the same BAR, so we want to restrict this ioremap from
878 * clobbering the GTT which we want ioremap_wc instead. Fortunately,
879 * the register BAR remains the same size for all the earlier
880 * generations up to Ironlake.
881 */
882 if (INTEL_INFO(dev)->gen < 5)
883 mmio_size = 512 * 1024;
884 else
885 mmio_size = 2 * 1024 * 1024;
52a05c30 886 dev_priv->regs = pci_iomap(pdev, mmio_bar, mmio_size);
0673ad47
CW		 887 if (dev_priv->regs == NULL) {
888 DRM_ERROR("failed to map registers\n");
889
890 return -EIO;
891 }
892
893 /* Try to make sure MCHBAR is enabled before poking at it */
894 intel_setup_mchbar(dev);
895
896 return 0;
897}
898
899static void i915_mmio_cleanup(struct drm_device *dev)
900{
901 struct drm_i915_private *dev_priv = to_i915(dev);
52a05c30 902 struct pci_dev *pdev = dev_priv->drm.pdev;
0673ad47
CW		 903
904 intel_teardown_mchbar(dev);
52a05c30 905 pci_iounmap(pdev, dev_priv->regs);
0673ad47
CW		 906}
907
908/**
909 * i915_driver_init_mmio - setup device MMIO
910 * @dev_priv: device private
911 *
912 * Setup minimal device state necessary for MMIO accesses later in the
913 * initialization sequence. The setup here should avoid any other device-wide
914 * side effects or exposing the driver via kernel internal or user space
915 * interfaces.
916 */
917static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
918{
91c8a326 919 struct drm_device *dev = &dev_priv->drm;
0673ad47
CW		 920 int ret;
921
922 if (i915_inject_load_failure())
923 return -ENODEV;
924
925 if (i915_get_bridge_dev(dev))
926 return -EIO;
927
928 ret = i915_mmio_setup(dev);
929 if (ret < 0)
930 goto put_bridge;
931
932 intel_uncore_init(dev_priv);
933
934 return 0;
935
936put_bridge:
937 pci_dev_put(dev_priv->bridge_dev);
938
939 return ret;
940}
941
942/**
943 * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio()
944 * @dev_priv: device private
945 */
946static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv)
947{
91c8a326 948 struct drm_device *dev = &dev_priv->drm;
0673ad47
CW		 949
950 intel_uncore_fini(dev_priv);
951 i915_mmio_cleanup(dev);
952 pci_dev_put(dev_priv->bridge_dev);
953}
954
94b4f3ba
CW		 955static void intel_sanitize_options(struct drm_i915_private *dev_priv)
956{
957 i915.enable_execlists =
958 intel_sanitize_enable_execlists(dev_priv,
959 i915.enable_execlists);
960
961 /*
962 * i915.enable_ppgtt is read-only, so do an early pass to validate the
963 * user's requested state against the hardware/driver capabilities. We
964 * do this now so that we can print out any log messages once rather
965 * than every time we check intel_enable_ppgtt().
966 */
967 i915.enable_ppgtt =
968 intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt);
969 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
39df9190
CW		 970
971 i915.semaphores = intel_sanitize_semaphores(dev_priv, i915.semaphores);
972 DRM_DEBUG_DRIVER("use GPU sempahores? %s\n", yesno(i915.semaphores));
94b4f3ba
CW		 973}
974
0673ad47
CW		 975/**
976 * i915_driver_init_hw - setup state requiring device access
977 * @dev_priv: device private
978 *
979 * Setup state that requires accessing the device, but doesn't require
980 * exposing the driver via kernel internal or userspace interfaces.
981 */
982static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
983{
52a05c30 984 struct pci_dev *pdev = dev_priv->drm.pdev;
91c8a326 985 struct drm_device *dev = &dev_priv->drm;
0673ad47
CW		 986 int ret;
987
988 if (i915_inject_load_failure())
989 return -ENODEV;
990
94b4f3ba
CW		 991 intel_device_info_runtime_init(dev_priv);
992
993 intel_sanitize_options(dev_priv);
0673ad47 994
97d6d7ab 995 ret = i915_ggtt_probe_hw(dev_priv);
0673ad47
CW		 996 if (ret)
997 return ret;
998
0673ad47
CW		 999 /* WARNING: Apparently we must kick fbdev drivers before vgacon,
1000 * otherwise the vga fbdev driver falls over. */
1001 ret = i915_kick_out_firmware_fb(dev_priv);
1002 if (ret) {
1003 DRM_ERROR("failed to remove conflicting framebuffer drivers\n");
1004 goto out_ggtt;
1005 }
1006
1007 ret = i915_kick_out_vgacon(dev_priv);
1008 if (ret) {
1009 DRM_ERROR("failed to remove conflicting VGA console\n");
1010 goto out_ggtt;
1011 }
1012
97d6d7ab 1013 ret = i915_ggtt_init_hw(dev_priv);
0088e522
CW		 1014 if (ret)
1015 return ret;
1016
97d6d7ab 1017 ret = i915_ggtt_enable_hw(dev_priv);
0088e522
CW		 1018 if (ret) {
1019 DRM_ERROR("failed to enable GGTT\n");
1020 goto out_ggtt;
1021 }
1022
52a05c30 1023 pci_set_master(pdev);
0673ad47
CW		 1024
1025 /* overlay on gen2 is broken and can't address above 1G */
1026 if (IS_GEN2(dev)) {
52a05c30 1027 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30));
0673ad47
CW		 1028 if (ret) {
1029 DRM_ERROR("failed to set DMA mask\n");
1030
1031 goto out_ggtt;
1032 }
1033 }
1034
0673ad47
CW		 1035 /* 965GM sometimes incorrectly writes to hardware status page (HWS)
1036 * using 32bit addressing, overwriting memory if HWS is located
1037 * above 4GB.
1038 *
1039 * The documentation also mentions an issue with undefined
1040 * behaviour if any general state is accessed within a page above 4GB,
1041 * which also needs to be handled carefully.
1042 */
1043 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) {
52a05c30 1044 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
0673ad47
CW		 1045
1046 if (ret) {
1047 DRM_ERROR("failed to set DMA mask\n");
1048
1049 goto out_ggtt;
1050 }
1051 }
1052
0673ad47
CW		 1053 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
1054 PM_QOS_DEFAULT_VALUE);
1055
1056 intel_uncore_sanitize(dev_priv);
1057
1058 intel_opregion_setup(dev_priv);
1059
1060 i915_gem_load_init_fences(dev_priv);
1061
1062 /* On the 945G/GM, the chipset reports the MSI capability on the
1063 * integrated graphics even though the support isn't actually there
1064 * according to the published specs. It doesn't appear to function
1065 * correctly in testing on 945G.
1066 * This may be a side effect of MSI having been made available for PEG
1067 * and the registers being closely associated.
1068 *
1069 * According to chipset errata, on the 965GM, MSI interrupts may
1070 * be lost or delayed, but we use them anyways to avoid
1071 * stuck interrupts on some machines.
1072 */
1073 if (!IS_I945G(dev) && !IS_I945GM(dev)) {
52a05c30 1074 if (pci_enable_msi(pdev) < 0)
0673ad47
CW		 1075 DRM_DEBUG_DRIVER("can't enable MSI");
1076 }
1077
1078 return 0;
1079
1080out_ggtt:
97d6d7ab 1081 i915_ggtt_cleanup_hw(dev_priv);
0673ad47
CW		 1082
1083 return ret;
1084}
1085
1086/**
1087 * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw()
1088 * @dev_priv: device private
1089 */
1090static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv)
1091{
52a05c30 1092 struct pci_dev *pdev = dev_priv->drm.pdev;
0673ad47 1093
52a05c30
DW		 1094 if (pdev->msi_enabled)
1095 pci_disable_msi(pdev);
0673ad47
CW		 1096
1097 pm_qos_remove_request(&dev_priv->pm_qos);
97d6d7ab 1098 i915_ggtt_cleanup_hw(dev_priv);
0673ad47
CW		 1099}
1100
1101/**
1102 * i915_driver_register - register the driver with the rest of the system
1103 * @dev_priv: device private
1104 *
1105 * Perform any steps necessary to make the driver available via kernel
1106 * internal or userspace interfaces.
1107 */
1108static void i915_driver_register(struct drm_i915_private *dev_priv)
1109{
91c8a326 1110 struct drm_device *dev = &dev_priv->drm;
0673ad47
CW		 1111
1112 i915_gem_shrinker_init(dev_priv);
1113
1114 /*
1115 * Notify a valid surface after modesetting,
1116 * when running inside a VM.
1117 */
1118 if (intel_vgpu_active(dev_priv))
1119 I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY);
1120
1121 /* Reveal our presence to userspace */
1122 if (drm_dev_register(dev, 0) == 0) {
1123 i915_debugfs_register(dev_priv);
694c2828 1124 i915_setup_sysfs(dev_priv);
0673ad47
CW		 1125 } else
1126 DRM_ERROR("Failed to register driver for userspace access!\n");
1127
1128 if (INTEL_INFO(dev_priv)->num_pipes) {
1129 /* Must be done after probing outputs */
1130 intel_opregion_register(dev_priv);
1131 acpi_video_register();
1132 }
1133
1134 if (IS_GEN5(dev_priv))
1135 intel_gpu_ips_init(dev_priv);
1136
1137 i915_audio_component_init(dev_priv);
1138
1139 /*
1140 * Some ports require correctly set-up hpd registers for detection to
1141 * work properly (leading to ghost connected connector status), e.g. VGA
1142 * on gm45. Hence we can only set up the initial fbdev config after hpd
1143 * irqs are fully enabled. We do it last so that the async config
1144 * cannot run before the connectors are registered.
1145 */
1146 intel_fbdev_initial_config_async(dev);
1147}
1148
1149/**
1150 * i915_driver_unregister - cleanup the registration done in i915_driver_regiser()
1151 * @dev_priv: device private
1152 */
1153static void i915_driver_unregister(struct drm_i915_private *dev_priv)
1154{
1155 i915_audio_component_cleanup(dev_priv);
1156
1157 intel_gpu_ips_teardown();
1158 acpi_video_unregister();
1159 intel_opregion_unregister(dev_priv);
1160
694c2828 1161 i915_teardown_sysfs(dev_priv);
0673ad47 1162 i915_debugfs_unregister(dev_priv);
91c8a326 1163 drm_dev_unregister(&dev_priv->drm);
0673ad47
CW		 1164
1165 i915_gem_shrinker_cleanup(dev_priv);
1166}
1167
1168/**
1169 * i915_driver_load - setup chip and create an initial config
1170 * @dev: DRM device
1171 * @flags: startup flags
1172 *
1173 * The driver load routine has to do several things:
1174 * - drive output discovery via intel_modeset_init()
1175 * - initialize the memory manager
1176 * - allocate initial config memory
1177 * - setup the DRM framebuffer with the allocated memory
1178 */
42f5551d 1179int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
0673ad47
CW		 1180{
1181 struct drm_i915_private *dev_priv;
1182 int ret;
7d87a7f7 1183
a09d0ba1
CW		 1184 if (i915.nuclear_pageflip)
1185 driver.driver_features |= DRIVER_ATOMIC;
1186
0673ad47
CW		 1187 ret = -ENOMEM;
1188 dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
1189 if (dev_priv)
1190 ret = drm_dev_init(&dev_priv->drm, &driver, &pdev->dev);
1191 if (ret) {
1192 dev_printk(KERN_ERR, &pdev->dev,
1193 "[" DRM_NAME ":%s] allocation failed\n", __func__);
1194 kfree(dev_priv);
1195 return ret;
1196 }
72bbf0af 1197
0673ad47
CW		 1198 dev_priv->drm.pdev = pdev;
1199 dev_priv->drm.dev_private = dev_priv;
719388e1 1200
0673ad47
CW		 1201 ret = pci_enable_device(pdev);
1202 if (ret)
1203 goto out_free_priv;
1347f5b4 1204
0673ad47 1205 pci_set_drvdata(pdev, &dev_priv->drm);
ef11bdb3 1206
0673ad47
CW		 1207 ret = i915_driver_init_early(dev_priv, ent);
1208 if (ret < 0)
1209 goto out_pci_disable;
ef11bdb3 1210
0673ad47 1211 intel_runtime_pm_get(dev_priv);
1da177e4 1212
0673ad47
CW		 1213 ret = i915_driver_init_mmio(dev_priv);
1214 if (ret < 0)
1215 goto out_runtime_pm_put;
79e53945 1216
0673ad47
CW		 1217 ret = i915_driver_init_hw(dev_priv);
1218 if (ret < 0)
1219 goto out_cleanup_mmio;
30c964a6
RB		 1220
1221 /*
0673ad47
CW		 1222 * TODO: move the vblank init and parts of modeset init steps into one
1223 * of the i915_driver_init_/i915_driver_register functions according
1224 * to the role/effect of the given init step.
30c964a6 1225 */
0673ad47 1226 if (INTEL_INFO(dev_priv)->num_pipes) {
91c8a326 1227 ret = drm_vblank_init(&dev_priv->drm,
0673ad47
CW		 1228 INTEL_INFO(dev_priv)->num_pipes);
1229 if (ret)
1230 goto out_cleanup_hw;
30c964a6
RB		 1231 }
1232
91c8a326 1233 ret = i915_load_modeset_init(&dev_priv->drm);
0673ad47
CW		 1234 if (ret < 0)
1235 goto out_cleanup_vblank;
1236
1237 i915_driver_register(dev_priv);
1238
1239 intel_runtime_pm_enable(dev_priv);
1240
bc5ca47c
CW		 1241 /* Everything is in place, we can now relax! */
1242 DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n",
1243 driver.name, driver.major, driver.minor, driver.patchlevel,
1244 driver.date, pci_name(pdev), dev_priv->drm.primary->index);
1245
0673ad47
CW		 1246 intel_runtime_pm_put(dev_priv);
1247
1248 return 0;
1249
1250out_cleanup_vblank:
91c8a326 1251 drm_vblank_cleanup(&dev_priv->drm);
0673ad47
CW		 1252out_cleanup_hw:
1253 i915_driver_cleanup_hw(dev_priv);
1254out_cleanup_mmio:
1255 i915_driver_cleanup_mmio(dev_priv);
1256out_runtime_pm_put:
1257 intel_runtime_pm_put(dev_priv);
1258 i915_driver_cleanup_early(dev_priv);
1259out_pci_disable:
1260 pci_disable_device(pdev);
1261out_free_priv:
1262 i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret);
1263 drm_dev_unref(&dev_priv->drm);
30c964a6
RB		 1264 return ret;
1265}
1266
42f5551d 1267void i915_driver_unload(struct drm_device *dev)
3bad0781 1268{
fac5e23e 1269 struct drm_i915_private *dev_priv = to_i915(dev);
52a05c30 1270 struct pci_dev *pdev = dev_priv->drm.pdev;
3bad0781 1271
0673ad47
CW		 1272 intel_fbdev_fini(dev);
1273
42f5551d
CW		 1274 if (i915_gem_suspend(dev))
1275 DRM_ERROR("failed to idle hardware; continuing to unload!\n");
ce1bb329 1276
0673ad47
CW		 1277 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
1278
1279 i915_driver_unregister(dev_priv);
1280
1281 drm_vblank_cleanup(dev);
1282
1283 intel_modeset_cleanup(dev);
1284
3bad0781 1285 /*
0673ad47
CW		 1286 * free the memory space allocated for the child device
1287 * config parsed from VBT
3bad0781 1288 */
0673ad47
CW		 1289 if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) {
1290 kfree(dev_priv->vbt.child_dev);
1291 dev_priv->vbt.child_dev = NULL;
1292 dev_priv->vbt.child_dev_num = 0;
1293 }
1294 kfree(dev_priv->vbt.sdvo_lvds_vbt_mode);
1295 dev_priv->vbt.sdvo_lvds_vbt_mode = NULL;
1296 kfree(dev_priv->vbt.lfp_lvds_vbt_mode);
1297 dev_priv->vbt.lfp_lvds_vbt_mode = NULL;
3bad0781 1298
52a05c30
DW		 1299 vga_switcheroo_unregister_client(pdev);
1300 vga_client_register(pdev, NULL, NULL, NULL);
bcdb72ac 1301
0673ad47 1302 intel_csr_ucode_fini(dev_priv);
bcdb72ac 1303
0673ad47
CW		 1304 /* Free error state after interrupts are fully disabled. */
1305 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
1306 i915_destroy_error_state(dev);
1307
1308 /* Flush any outstanding unpin_work. */
b7137e0c 1309 drain_workqueue(dev_priv->wq);
0673ad47
CW		 1310
1311 intel_guc_fini(dev);
1312 i915_gem_fini(dev);
1313 intel_fbc_cleanup_cfb(dev_priv);
1314
1315 intel_power_domains_fini(dev_priv);
1316
1317 i915_driver_cleanup_hw(dev_priv);
1318 i915_driver_cleanup_mmio(dev_priv);
1319
1320 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
1321
1322 i915_driver_cleanup_early(dev_priv);
3bad0781
ZW		 1323}
1324
0673ad47 1325static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
2911a35b 1326{
0673ad47 1327 int ret;
2911a35b 1328
0673ad47
CW		 1329 ret = i915_gem_open(dev, file);
1330 if (ret)
1331 return ret;
2911a35b 1332
0673ad47
CW		 1333 return 0;
1334}
71386ef9 1335
0673ad47
CW		 1336/**
1337 * i915_driver_lastclose - clean up after all DRM clients have exited
1338 * @dev: DRM device
1339 *
1340 * Take care of cleaning up after all DRM clients have exited. In the
1341 * mode setting case, we want to restore the kernel's initial mode (just
1342 * in case the last client left us in a bad state).
1343 *
1344 * Additionally, in the non-mode setting case, we'll tear down the GTT
1345 * and DMA structures, since the kernel won't be using them, and clea
1346 * up any GEM state.
1347 */
1348static void i915_driver_lastclose(struct drm_device *dev)
1349{
1350 intel_fbdev_restore_mode(dev);
1351 vga_switcheroo_process_delayed_switch();
1352}
2911a35b 1353
0673ad47
CW		 1354static void i915_driver_preclose(struct drm_device *dev, struct drm_file *file)
1355{
1356 mutex_lock(&dev->struct_mutex);
1357 i915_gem_context_close(dev, file);
1358 i915_gem_release(dev, file);
1359 mutex_unlock(&dev->struct_mutex);
1360}
1361
1362static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
1363{
1364 struct drm_i915_file_private *file_priv = file->driver_priv;
1365
1366 kfree(file_priv);
2911a35b
BW		 1367}
1368
07f9cd0b
ID		 1369static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
1370{
91c8a326 1371 struct drm_device *dev = &dev_priv->drm;
19c8054c 1372 struct intel_encoder *encoder;
07f9cd0b
ID		 1373
1374 drm_modeset_lock_all(dev);
19c8054c
JN		 1375 for_each_intel_encoder(dev, encoder)
1376 if (encoder->suspend)
1377 encoder->suspend(encoder);
07f9cd0b
ID		 1378 drm_modeset_unlock_all(dev);
1379}
1380
1a5df187
PZ		 1381static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
1382 bool rpm_resume);
507e126e 1383static int vlv_suspend_complete(struct drm_i915_private *dev_priv);
f75a1985 1384
bc87229f
ID		 1385static bool suspend_to_idle(struct drm_i915_private *dev_priv)
1386{
1387#if IS_ENABLED(CONFIG_ACPI_SLEEP)
1388 if (acpi_target_system_state() < ACPI_STATE_S3)
1389 return true;
1390#endif
1391 return false;
1392}
ebc32824 1393
5e365c39 1394static int i915_drm_suspend(struct drm_device *dev)
ba8bbcf6 1395{
fac5e23e 1396 struct drm_i915_private *dev_priv = to_i915(dev);
52a05c30 1397 struct pci_dev *pdev = dev_priv->drm.pdev;
e5747e3a 1398 pci_power_t opregion_target_state;
d5818938 1399 int error;
61caf87c 1400
b8efb17b
ZR		 1401 /* ignore lid events during suspend */
1402 mutex_lock(&dev_priv->modeset_restore_lock);
1403 dev_priv->modeset_restore = MODESET_SUSPENDED;
1404 mutex_unlock(&dev_priv->modeset_restore_lock);
1405
1f814dac
ID		 1406 disable_rpm_wakeref_asserts(dev_priv);
1407
c67a470b
PZ		 1408 /* We do a lot of poking in a lot of registers, make sure they work
1409 * properly. */
da7e29bd 1410 intel_display_set_init_power(dev_priv, true);
cb10799c 1411
5bcf719b
DA		 1412 drm_kms_helper_poll_disable(dev);
1413
52a05c30 1414 pci_save_state(pdev);
ba8bbcf6 1415
d5818938
DV		 1416 error = i915_gem_suspend(dev);
1417 if (error) {
52a05c30 1418 dev_err(&pdev->dev,
d5818938 1419 "GEM idle failed, resume might fail\n");
1f814dac 1420 goto out;
d5818938 1421 }
db1b76ca 1422
a1c41994
AD		 1423 intel_guc_suspend(dev);
1424
6b72d486 1425 intel_display_suspend(dev);
2eb5252e 1426
d5818938 1427 intel_dp_mst_suspend(dev);
7d708ee4 1428
d5818938
DV		 1429 intel_runtime_pm_disable_interrupts(dev_priv);
1430 intel_hpd_cancel_work(dev_priv);
09b64267 1431
d5818938 1432 intel_suspend_encoders(dev_priv);
0e32b39c 1433
d5818938 1434 intel_suspend_hw(dev);
5669fcac 1435
828c7908
BW		 1436 i915_gem_suspend_gtt_mappings(dev);
1437
9e06dd39
JB		 1438 i915_save_state(dev);
1439
bc87229f 1440 opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
6f9f4b7a 1441 intel_opregion_notify_adapter(dev_priv, opregion_target_state);
e5747e3a 1442
dc97997a 1443 intel_uncore_forcewake_reset(dev_priv, false);
03d92e47 1444 intel_opregion_unregister(dev_priv);
8ee1c3db 1445
82e3b8c1 1446 intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
3fa016a0 1447
62d5d69b
MK		 1448 dev_priv->suspend_count++;
1449
85e90679
KCA		 1450 intel_display_set_init_power(dev_priv, false);
1451
f74ed08d 1452 intel_csr_ucode_suspend(dev_priv);
f514c2d8 1453
1f814dac
ID		 1454out:
1455 enable_rpm_wakeref_asserts(dev_priv);
1456
1457 return error;
84b79f8d
RW		 1458}
1459
c49d13ee 1460static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
c3c09c95 1461{
c49d13ee 1462 struct drm_i915_private *dev_priv = to_i915(dev);
52a05c30 1463 struct pci_dev *pdev = dev_priv->drm.pdev;
bc87229f 1464 bool fw_csr;
c3c09c95
ID		 1465 int ret;
1466
1f814dac
ID		 1467 disable_rpm_wakeref_asserts(dev_priv);
1468
a7c8125f
ID		 1469 fw_csr = !IS_BROXTON(dev_priv) &&
1470 suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload;
bc87229f
ID		 1471 /*
1472 * In case of firmware assisted context save/restore don't manually
1473 * deinit the power domains. This also means the CSR/DMC firmware will
1474 * stay active, it will power down any HW resources as required and
1475 * also enable deeper system power states that would be blocked if the
1476 * firmware was inactive.
1477 */
1478 if (!fw_csr)
1479 intel_power_domains_suspend(dev_priv);
73dfc227 1480
507e126e 1481 ret = 0;
b8aea3d1 1482 if (IS_BROXTON(dev_priv))
507e126e 1483 bxt_enable_dc9(dev_priv);
b8aea3d1 1484 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
507e126e
ID		 1485 hsw_enable_pc8(dev_priv);
1486 else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1487 ret = vlv_suspend_complete(dev_priv);
c3c09c95
ID		 1488
1489 if (ret) {
1490 DRM_ERROR("Suspend complete failed: %d\n", ret);
bc87229f
ID		 1491 if (!fw_csr)
1492 intel_power_domains_init_hw(dev_priv, true);
c3c09c95 1493
1f814dac 1494 goto out;
c3c09c95
ID		 1495 }
1496
52a05c30 1497 pci_disable_device(pdev);
ab3be73f 1498 /*
54875571 1499 * During hibernation on some platforms the BIOS may try to access
ab3be73f
ID		 1500 * the device even though it's already in D3 and hang the machine. So
1501 * leave the device in D0 on those platforms and hope the BIOS will
54875571
ID		 1502 * power down the device properly. The issue was seen on multiple old
1503 * GENs with different BIOS vendors, so having an explicit blacklist
1504 * is inpractical; apply the workaround on everything pre GEN6. The
1505 * platforms where the issue was seen:
1506 * Lenovo Thinkpad X301, X61s, X60, T60, X41
1507 * Fujitsu FSC S7110
1508 * Acer Aspire 1830T
ab3be73f 1509 */
54875571 1510 if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6))
52a05c30 1511 pci_set_power_state(pdev, PCI_D3hot);
c3c09c95 1512
bc87229f
ID		 1513 dev_priv->suspended_to_idle = suspend_to_idle(dev_priv);
1514
1f814dac
ID		 1515out:
1516 enable_rpm_wakeref_asserts(dev_priv);
1517
1518 return ret;
c3c09c95
ID		 1519}
1520
1751fcf9 1521int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state)
84b79f8d
RW		 1522{
1523 int error;
1524
ded8b07d 1525 if (!dev) {
84b79f8d
RW		 1526 DRM_ERROR("dev: %p\n", dev);
1527 DRM_ERROR("DRM not initialized, aborting suspend.\n");
1528 return -ENODEV;
1529 }
1530
0b14cbd2
ID		 1531 if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND &&
1532 state.event != PM_EVENT_FREEZE))
1533 return -EINVAL;
5bcf719b
DA		 1534
1535 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1536 return 0;
6eecba33 1537
5e365c39 1538 error = i915_drm_suspend(dev);
84b79f8d
RW		 1539 if (error)
1540 return error;
1541
ab3be73f 1542 return i915_drm_suspend_late(dev, false);
ba8bbcf6
JB		 1543}
1544
5e365c39 1545static int i915_drm_resume(struct drm_device *dev)
76c4b250 1546{
fac5e23e 1547 struct drm_i915_private *dev_priv = to_i915(dev);
ac840ae5 1548 int ret;
9d49c0ef 1549
1f814dac 1550 disable_rpm_wakeref_asserts(dev_priv);
abc80abd 1551 intel_sanitize_gt_powersave(dev_priv);
1f814dac 1552
97d6d7ab 1553 ret = i915_ggtt_enable_hw(dev_priv);
ac840ae5
VS		 1554 if (ret)
1555 DRM_ERROR("failed to re-enable GGTT\n");
1556
f74ed08d
ID		 1557 intel_csr_ucode_resume(dev_priv);
1558
5ab57c70 1559 i915_gem_resume(dev);
9d49c0ef 1560
61caf87c 1561 i915_restore_state(dev);
8090ba8c 1562 intel_pps_unlock_regs_wa(dev_priv);
6f9f4b7a 1563 intel_opregion_setup(dev_priv);
61caf87c 1564
d5818938
DV		 1565 intel_init_pch_refclk(dev);
1566 drm_mode_config_reset(dev);
1833b134 1567
364aece0
PA		 1568 /*
1569 * Interrupts have to be enabled before any batches are run. If not the
1570 * GPU will hang. i915_gem_init_hw() will initiate batches to
1571 * update/restore the context.
1572 *
1573 * Modeset enabling in intel_modeset_init_hw() also needs working
1574 * interrupts.
1575 */
1576 intel_runtime_pm_enable_interrupts(dev_priv);
1577
d5818938
DV		 1578 mutex_lock(&dev->struct_mutex);
1579 if (i915_gem_init_hw(dev)) {
1580 DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
821ed7df 1581 i915_gem_set_wedged(dev_priv);
d5818938
DV		 1582 }
1583 mutex_unlock(&dev->struct_mutex);
226485e9 1584
a1c41994
AD		 1585 intel_guc_resume(dev);
1586
d5818938 1587 intel_modeset_init_hw(dev);
24576d23 1588
d5818938
DV		 1589 spin_lock_irq(&dev_priv->irq_lock);
1590 if (dev_priv->display.hpd_irq_setup)
91d14251 1591 dev_priv->display.hpd_irq_setup(dev_priv);
d5818938 1592 spin_unlock_irq(&dev_priv->irq_lock);
0e32b39c 1593
d5818938 1594 intel_dp_mst_resume(dev);
e7d6f7d7 1595
a16b7658
L		 1596 intel_display_resume(dev);
1597
d5818938
DV		 1598 /*
1599 * ... but also need to make sure that hotplug processing
1600 * doesn't cause havoc. Like in the driver load code we don't
1601 * bother with the tiny race here where we might loose hotplug
1602 * notifications.
1603 * */
1604 intel_hpd_init(dev_priv);
1605 /* Config may have changed between suspend and resume */
1606 drm_helper_hpd_irq_event(dev);
1daed3fb 1607
03d92e47 1608 intel_opregion_register(dev_priv);
44834a67 1609
82e3b8c1 1610 intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
073f34d9 1611
b8efb17b
ZR		 1612 mutex_lock(&dev_priv->modeset_restore_lock);
1613 dev_priv->modeset_restore = MODESET_DONE;
1614 mutex_unlock(&dev_priv->modeset_restore_lock);
8a187455 1615
6f9f4b7a 1616 intel_opregion_notify_adapter(dev_priv, PCI_D0);
e5747e3a 1617
54b4f68f 1618 intel_autoenable_gt_powersave(dev_priv);
ee6f280e
ID		 1619 drm_kms_helper_poll_enable(dev);
1620
1f814dac
ID		 1621 enable_rpm_wakeref_asserts(dev_priv);
1622
074c6ada 1623 return 0;
84b79f8d
RW		 1624}
1625
5e365c39 1626static int i915_drm_resume_early(struct drm_device *dev)
84b79f8d 1627{
fac5e23e 1628 struct drm_i915_private *dev_priv = to_i915(dev);
52a05c30 1629 struct pci_dev *pdev = dev_priv->drm.pdev;
44410cd0 1630 int ret;
36d61e67 1631
76c4b250
ID		 1632 /*
1633 * We have a resume ordering issue with the snd-hda driver also
1634 * requiring our device to be power up. Due to the lack of a
1635 * parent/child relationship we currently solve this with an early
1636 * resume hook.
1637 *
1638 * FIXME: This should be solved with a special hdmi sink device or
1639 * similar so that power domains can be employed.
1640 */
44410cd0
ID		 1641
1642 /*
1643 * Note that we need to set the power state explicitly, since we
1644 * powered off the device during freeze and the PCI core won't power
1645 * it back up for us during thaw. Powering off the device during
1646 * freeze is not a hard requirement though, and during the
1647 * suspend/resume phases the PCI core makes sure we get here with the
1648 * device powered on. So in case we change our freeze logic and keep
1649 * the device powered we can also remove the following set power state
1650 * call.
1651 */
52a05c30 1652 ret = pci_set_power_state(pdev, PCI_D0);
44410cd0
ID		 1653 if (ret) {
1654 DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret);
1655 goto out;
1656 }
1657
1658 /*
1659 * Note that pci_enable_device() first enables any parent bridge
1660 * device and only then sets the power state for this device. The
1661 * bridge enabling is a nop though, since bridge devices are resumed
1662 * first. The order of enabling power and enabling the device is
1663 * imposed by the PCI core as described above, so here we preserve the
1664 * same order for the freeze/thaw phases.
1665 *
1666 * TODO: eventually we should remove pci_disable_device() /
1667 * pci_enable_enable_device() from suspend/resume. Due to how they
1668 * depend on the device enable refcount we can't anyway depend on them
1669 * disabling/enabling the device.
1670 */
52a05c30 1671 if (pci_enable_device(pdev)) {
bc87229f
ID		 1672 ret = -EIO;
1673 goto out;
1674 }
84b79f8d 1675
52a05c30 1676 pci_set_master(pdev);
84b79f8d 1677
1f814dac
ID		 1678 disable_rpm_wakeref_asserts(dev_priv);
1679
666a4537 1680 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1a5df187 1681 ret = vlv_resume_prepare(dev_priv, false);
36d61e67 1682 if (ret)
ff0b187f
DL		 1683 DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
1684 ret);
36d61e67 1685
dc97997a 1686 intel_uncore_early_sanitize(dev_priv, true);
efee833a 1687
dc97997a 1688 if (IS_BROXTON(dev_priv)) {
da2f41d1
ID		 1689 if (!dev_priv->suspended_to_idle)
1690 gen9_sanitize_dc_state(dev_priv);
507e126e 1691 bxt_disable_dc9(dev_priv);
da2f41d1 1692 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
a9a6b73a 1693 hsw_disable_pc8(dev_priv);
da2f41d1 1694 }
efee833a 1695
dc97997a 1696 intel_uncore_sanitize(dev_priv);
bc87229f 1697
a7c8125f
ID		 1698 if (IS_BROXTON(dev_priv) ||
1699 !(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload))
bc87229f
ID		 1700 intel_power_domains_init_hw(dev_priv, true);
1701
6e35e8ab
ID		 1702 enable_rpm_wakeref_asserts(dev_priv);
1703
bc87229f
ID		 1704out:
1705 dev_priv->suspended_to_idle = false;
36d61e67
ID		 1706
1707 return ret;
76c4b250
ID		 1708}
1709
1751fcf9 1710int i915_resume_switcheroo(struct drm_device *dev)
76c4b250 1711{
50a0072f 1712 int ret;
76c4b250 1713
097dd837
ID		 1714 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1715 return 0;
1716
5e365c39 1717 ret = i915_drm_resume_early(dev);
50a0072f
ID		 1718 if (ret)
1719 return ret;
1720
5a17514e
ID		 1721 return i915_drm_resume(dev);
1722}
1723
11ed50ec 1724/**
f3953dcb 1725 * i915_reset - reset chip after a hang
11ed50ec 1726 * @dev: drm device to reset
11ed50ec 1727 *
780f262a
CW		 1728 * Reset the chip. Useful if a hang is detected. Marks the device as wedged
1729 * on failure.
11ed50ec 1730 *
221fe799
CW		 1731 * Caller must hold the struct_mutex.
1732 *
11ed50ec
BG		 1733 * Procedure is fairly simple:
1734 * - reset the chip using the reset reg
1735 * - re-init context state
1736 * - re-init hardware status page
1737 * - re-init ring buffer
1738 * - re-init interrupt state
1739 * - re-init display
1740 */
780f262a 1741void i915_reset(struct drm_i915_private *dev_priv)
11ed50ec 1742{
91c8a326 1743 struct drm_device *dev = &dev_priv->drm;
d98c52cf 1744 struct i915_gpu_error *error = &dev_priv->gpu_error;
0573ed4a 1745 int ret;
11ed50ec 1746
221fe799
CW		 1747 lockdep_assert_held(&dev->struct_mutex);
1748
1749 if (!test_and_clear_bit(I915_RESET_IN_PROGRESS, &error->flags))
780f262a 1750 return;
11ed50ec 1751
d98c52cf 1752 /* Clear any previous failed attempts at recovery. Time to try again. */
8af29b0c
CW		 1753 __clear_bit(I915_WEDGED, &error->flags);
1754 error->reset_count++;
d98c52cf 1755
7b4d3a16 1756 pr_notice("drm/i915: Resetting chip after gpu hang\n");
dc97997a 1757 ret = intel_gpu_reset(dev_priv, ALL_ENGINES);
0573ed4a 1758 if (ret) {
804e59a8
CW		 1759 if (ret != -ENODEV)
1760 DRM_ERROR("Failed to reset chip: %i\n", ret);
1761 else
1762 DRM_DEBUG_DRIVER("GPU reset disabled\n");
d98c52cf 1763 goto error;
11ed50ec
BG		 1764 }
1765
821ed7df 1766 i915_gem_reset(dev_priv);
1362b776
VS		 1767 intel_overlay_reset(dev_priv);
1768
11ed50ec
BG		 1769 /* Ok, now get things going again... */
1770
1771 /*
1772 * Everything depends on having the GTT running, so we need to start
1773 * there. Fortunately we don't need to do this unless we reset the
1774 * chip at a PCI level.
1775 *
1776 * Next we need to restore the context, but we don't use those
1777 * yet either...
1778 *
1779 * Ring buffer needs to be re-initialized in the KMS case, or if X
1780 * was running at the time of the reset (i.e. we weren't VT
1781 * switched away).
1782 */
33d30a9c 1783 ret = i915_gem_init_hw(dev);
33d30a9c
DV		 1784 if (ret) {
1785 DRM_ERROR("Failed hw init on reset %d\n", ret);
d98c52cf 1786 goto error;
11ed50ec
BG		 1787 }
1788
780f262a
CW		 1789wakeup:
1790 wake_up_bit(&error->flags, I915_RESET_IN_PROGRESS);
1791 return;
d98c52cf
CW		 1792
1793error:
821ed7df 1794 i915_gem_set_wedged(dev_priv);
780f262a 1795 goto wakeup;
11ed50ec
BG		 1796}
1797
c49d13ee 1798static int i915_pm_suspend(struct device *kdev)
112b715e 1799{
c49d13ee
DW		 1800 struct pci_dev *pdev = to_pci_dev(kdev);
1801 struct drm_device *dev = pci_get_drvdata(pdev);
112b715e 1802
c49d13ee
DW		 1803 if (!dev) {
1804 dev_err(kdev, "DRM not initialized, aborting suspend.\n");
84b79f8d
RW		 1805 return -ENODEV;
1806 }
112b715e 1807
c49d13ee 1808 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
5bcf719b
DA		 1809 return 0;
1810
c49d13ee 1811 return i915_drm_suspend(dev);
76c4b250
ID		 1812}
1813
c49d13ee 1814static int i915_pm_suspend_late(struct device *kdev)
76c4b250 1815{
c49d13ee 1816 struct drm_device *dev = &kdev_to_i915(kdev)->drm;
76c4b250
ID		 1817
1818 /*
c965d995 1819 * We have a suspend ordering issue with the snd-hda driver also
76c4b250
ID		 1820 * requiring our device to be power up. Due to the lack of a
1821 * parent/child relationship we currently solve this with an late
1822 * suspend hook.
1823 *
1824 * FIXME: This should be solved with a special hdmi sink device or
1825 * similar so that power domains can be employed.
1826 */
c49d13ee 1827 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
76c4b250 1828 return 0;
112b715e 1829
c49d13ee 1830 return i915_drm_suspend_late(dev, false);
ab3be73f
ID		 1831}
1832
c49d13ee 1833static int i915_pm_poweroff_late(struct device *kdev)
ab3be73f 1834{
c49d13ee 1835 struct drm_device *dev = &kdev_to_i915(kdev)->drm;
ab3be73f 1836
c49d13ee 1837 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
ab3be73f
ID		 1838 return 0;
1839
c49d13ee 1840 return i915_drm_suspend_late(dev, true);
cbda12d7
ZW		 1841}
1842
c49d13ee 1843static int i915_pm_resume_early(struct device *kdev)
76c4b250 1844{
c49d13ee 1845 struct drm_device *dev = &kdev_to_i915(kdev)->drm;
76c4b250 1846
c49d13ee 1847 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
097dd837
ID		 1848 return 0;
1849
c49d13ee 1850 return i915_drm_resume_early(dev);
76c4b250
ID		 1851}
1852
c49d13ee 1853static int i915_pm_resume(struct device *kdev)
cbda12d7 1854{
c49d13ee 1855 struct drm_device *dev = &kdev_to_i915(kdev)->drm;
84b79f8d 1856
c49d13ee 1857 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
097dd837
ID		 1858 return 0;
1859
c49d13ee 1860 return i915_drm_resume(dev);
cbda12d7
ZW		 1861}
1862
1f19ac2a 1863/* freeze: before creating the hibernation_image */
c49d13ee 1864static int i915_pm_freeze(struct device *kdev)
1f19ac2a 1865{
c49d13ee 1866 return i915_pm_suspend(kdev);
1f19ac2a
CW		 1867}
1868
c49d13ee 1869static int i915_pm_freeze_late(struct device *kdev)
1f19ac2a 1870{
461fb99c
CW		 1871 int ret;
1872
c49d13ee 1873 ret = i915_pm_suspend_late(kdev);
461fb99c
CW		 1874 if (ret)
1875 return ret;
1876
c49d13ee 1877 ret = i915_gem_freeze_late(kdev_to_i915(kdev));
461fb99c
CW		 1878 if (ret)
1879 return ret;
1880
1881 return 0;
1f19ac2a
CW		 1882}
1883
1884/* thaw: called after creating the hibernation image, but before turning off. */
c49d13ee 1885static int i915_pm_thaw_early(struct device *kdev)
1f19ac2a 1886{
c49d13ee 1887 return i915_pm_resume_early(kdev);
1f19ac2a
CW		 1888}
1889
c49d13ee 1890static int i915_pm_thaw(struct device *kdev)
1f19ac2a 1891{
c49d13ee 1892 return i915_pm_resume(kdev);
1f19ac2a
CW		 1893}
1894
1895/* restore: called after loading the hibernation image. */
c49d13ee 1896static int i915_pm_restore_early(struct device *kdev)
1f19ac2a 1897{
c49d13ee 1898 return i915_pm_resume_early(kdev);
1f19ac2a
CW		 1899}
1900
c49d13ee 1901static int i915_pm_restore(struct device *kdev)
1f19ac2a 1902{
c49d13ee 1903 return i915_pm_resume(kdev);
1f19ac2a
CW		 1904}
1905
ddeea5b0
ID		 1906/*
1907 * Save all Gunit registers that may be lost after a D3 and a subsequent
1908 * S0i[R123] transition. The list of registers needing a save/restore is
1909 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
1910 * registers in the following way:
1911 * - Driver: saved/restored by the driver
1912 * - Punit : saved/restored by the Punit firmware
1913 * - No, w/o marking: no need to save/restore, since the register is R/O or
1914 * used internally by the HW in a way that doesn't depend
1915 * keeping the content across a suspend/resume.
1916 * - Debug : used for debugging
1917 *
1918 * We save/restore all registers marked with 'Driver', with the following
1919 * exceptions:
1920 * - Registers out of use, including also registers marked with 'Debug'.
1921 * These have no effect on the driver's operation, so we don't save/restore
1922 * them to reduce the overhead.
1923 * - Registers that are fully setup by an initialization function called from
1924 * the resume path. For example many clock gating and RPS/RC6 registers.
1925 * - Registers that provide the right functionality with their reset defaults.
1926 *
1927 * TODO: Except for registers that based on the above 3 criteria can be safely
1928 * ignored, we save/restore all others, practically treating the HW context as
1929 * a black-box for the driver. Further investigation is needed to reduce the
1930 * saved/restored registers even further, by following the same 3 criteria.
1931 */
1932static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv)
1933{
1934 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
1935 int i;
1936
1937 /* GAM 0x4000-0x4770 */
1938 s->wr_watermark = I915_READ(GEN7_WR_WATERMARK);
1939 s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL);
1940 s->arb_mode = I915_READ(ARB_MODE);
1941 s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0);
1942 s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1);
1943
1944 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
22dfe79f 1945 s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
ddeea5b0
ID		 1946
1947 s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
b5f1c97f 1948 s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT);
ddeea5b0
ID		 1949
1950 s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7);
1951 s->ecochk = I915_READ(GAM_ECOCHK);
1952 s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7);
1953 s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7);
1954
1955 s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR);
1956
1957 /* MBC 0x9024-0x91D0, 0x8500 */
1958 s->g3dctl = I915_READ(VLV_G3DCTL);
1959 s->gsckgctl = I915_READ(VLV_GSCKGCTL);
1960 s->mbctl = I915_READ(GEN6_MBCTL);
1961
1962 /* GCP 0x9400-0x9424, 0x8100-0x810C */
1963 s->ucgctl1 = I915_READ(GEN6_UCGCTL1);
1964 s->ucgctl3 = I915_READ(GEN6_UCGCTL3);
1965 s->rcgctl1 = I915_READ(GEN6_RCGCTL1);
1966 s->rcgctl2 = I915_READ(GEN6_RCGCTL2);
1967 s->rstctl = I915_READ(GEN6_RSTCTL);
1968 s->misccpctl = I915_READ(GEN7_MISCCPCTL);
1969
1970 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
1971 s->gfxpause = I915_READ(GEN6_GFXPAUSE);
1972 s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC);
1973 s->rpdeuc = I915_READ(GEN6_RPDEUC);
1974 s->ecobus = I915_READ(ECOBUS);
1975 s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL);
1976 s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT);
1977 s->rp_deucsw = I915_READ(GEN6_RPDEUCSW);
1978 s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR);
1979 s->rcedata = I915_READ(VLV_RCEDATA);
1980 s->spare2gh = I915_READ(VLV_SPAREG2H);
1981
1982 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
1983 s->gt_imr = I915_READ(GTIMR);
1984 s->gt_ier = I915_READ(GTIER);
1985 s->pm_imr = I915_READ(GEN6_PMIMR);
1986 s->pm_ier = I915_READ(GEN6_PMIER);
1987
1988 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
22dfe79f 1989 s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
ddeea5b0
ID		 1990
1991 /* GT SA CZ domain, 0x100000-0x138124 */
1992 s->tilectl = I915_READ(TILECTL);
1993 s->gt_fifoctl = I915_READ(GTFIFOCTL);
1994 s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL);
1995 s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
1996 s->pmwgicz = I915_READ(VLV_PMWGICZ);
1997
1998 /* Gunit-Display CZ domain, 0x182028-0x1821CF */
1999 s->gu_ctl0 = I915_READ(VLV_GU_CTL0);
2000 s->gu_ctl1 = I915_READ(VLV_GU_CTL1);
9c25210f 2001 s->pcbr = I915_READ(VLV_PCBR);
ddeea5b0
ID		 2002 s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2);
2003
2004 /*
2005 * Not saving any of:
2006 * DFT, 0x9800-0x9EC0
2007 * SARB, 0xB000-0xB1FC
2008 * GAC, 0x5208-0x524C, 0x14000-0x14C000
2009 * PCI CFG
2010 */
2011}
2012
2013static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv)
2014{
2015 struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
2016 u32 val;
2017 int i;
2018
2019 /* GAM 0x4000-0x4770 */
2020 I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark);
2021 I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl);
2022 I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16));
2023 I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0);
2024 I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1);
2025
2026 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
22dfe79f 2027 I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
ddeea5b0
ID		 2028
2029 I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
b5f1c97f 2030 I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
ddeea5b0
ID		 2031
2032 I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp);
2033 I915_WRITE(GAM_ECOCHK, s->ecochk);
2034 I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp);
2035 I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp);
2036
2037 I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr);
2038
2039 /* MBC 0x9024-0x91D0, 0x8500 */
2040 I915_WRITE(VLV_G3DCTL, s->g3dctl);
2041 I915_WRITE(VLV_GSCKGCTL, s->gsckgctl);
2042 I915_WRITE(GEN6_MBCTL, s->mbctl);
2043
2044 /* GCP 0x9400-0x9424, 0x8100-0x810C */
2045 I915_WRITE(GEN6_UCGCTL1, s->ucgctl1);
2046 I915_WRITE(GEN6_UCGCTL3, s->ucgctl3);
2047 I915_WRITE(GEN6_RCGCTL1, s->rcgctl1);
2048 I915_WRITE(GEN6_RCGCTL2, s->rcgctl2);
2049 I915_WRITE(GEN6_RSTCTL, s->rstctl);
2050 I915_WRITE(GEN7_MISCCPCTL, s->misccpctl);
2051
2052 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
2053 I915_WRITE(GEN6_GFXPAUSE, s->gfxpause);
2054 I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc);
2055 I915_WRITE(GEN6_RPDEUC, s->rpdeuc);
2056 I915_WRITE(ECOBUS, s->ecobus);
2057 I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl);
2058 I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout);
2059 I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw);
2060 I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr);
2061 I915_WRITE(VLV_RCEDATA, s->rcedata);
2062 I915_WRITE(VLV_SPAREG2H, s->spare2gh);
2063
2064 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
2065 I915_WRITE(GTIMR, s->gt_imr);
2066 I915_WRITE(GTIER, s->gt_ier);
2067 I915_WRITE(GEN6_PMIMR, s->pm_imr);
2068 I915_WRITE(GEN6_PMIER, s->pm_ier);
2069
2070 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
22dfe79f 2071 I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
ddeea5b0
ID		 2072
2073 /* GT SA CZ domain, 0x100000-0x138124 */
2074 I915_WRITE(TILECTL, s->tilectl);
2075 I915_WRITE(GTFIFOCTL, s->gt_fifoctl);
2076 /*
2077 * Preserve the GT allow wake and GFX force clock bit, they are not
2078 * be restored, as they are used to control the s0ix suspend/resume
2079 * sequence by the caller.
2080 */
2081 val = I915_READ(VLV_GTLC_WAKE_CTRL);
2082 val &= VLV_GTLC_ALLOWWAKEREQ;
2083 val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
2084 I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2085
2086 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2087 val &= VLV_GFX_CLK_FORCE_ON_BIT;
2088 val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
2089 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2090
2091 I915_WRITE(VLV_PMWGICZ, s->pmwgicz);
2092
2093 /* Gunit-Display CZ domain, 0x182028-0x1821CF */
2094 I915_WRITE(VLV_GU_CTL0, s->gu_ctl0);
2095 I915_WRITE(VLV_GU_CTL1, s->gu_ctl1);
9c25210f 2096 I915_WRITE(VLV_PCBR, s->pcbr);
ddeea5b0
ID		 2097 I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2);
2098}
2099
650ad970
ID		 2100int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
2101{
2102 u32 val;
2103 int err;
2104
650ad970
ID		 2105 val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2106 val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
2107 if (force_on)
2108 val |= VLV_GFX_CLK_FORCE_ON_BIT;
2109 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2110
2111 if (!force_on)
2112 return 0;
2113
c6ddc5f3
CW		 2114 err = intel_wait_for_register(dev_priv,
2115 VLV_GTLC_SURVIVABILITY_REG,
2116 VLV_GFX_CLK_STATUS_BIT,
2117 VLV_GFX_CLK_STATUS_BIT,
2118 20);
650ad970
ID		 2119 if (err)
2120 DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n",
2121 I915_READ(VLV_GTLC_SURVIVABILITY_REG));
2122
2123 return err;
650ad970
ID		 2124}
2125
ddeea5b0
ID		 2126static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
2127{
2128 u32 val;
2129 int err = 0;
2130
2131 val = I915_READ(VLV_GTLC_WAKE_CTRL);
2132 val &= ~VLV_GTLC_ALLOWWAKEREQ;
2133 if (allow)
2134 val |= VLV_GTLC_ALLOWWAKEREQ;
2135 I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2136 POSTING_READ(VLV_GTLC_WAKE_CTRL);
2137
b2736695
CW		 2138 err = intel_wait_for_register(dev_priv,
2139 VLV_GTLC_PW_STATUS,
2140 VLV_GTLC_ALLOWWAKEACK,
2141 allow,
2142 1);
ddeea5b0
ID		 2143 if (err)
2144 DRM_ERROR("timeout disabling GT waking\n");
b2736695 2145
ddeea5b0 2146 return err;
ddeea5b0
ID		 2147}
2148
2149static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
2150 bool wait_for_on)
2151{
2152 u32 mask;
2153 u32 val;
2154 int err;
2155
2156 mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
2157 val = wait_for_on ? mask : 0;
41ce405e 2158 if ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val)
ddeea5b0
ID		 2159 return 0;
2160
2161 DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n",
87ad3212
JN		 2162 onoff(wait_for_on),
2163 I915_READ(VLV_GTLC_PW_STATUS));
ddeea5b0
ID		 2164
2165 /*
2166 * RC6 transitioning can be delayed up to 2 msec (see
2167 * valleyview_enable_rps), use 3 msec for safety.
2168 */
41ce405e
CW		 2169 err = intel_wait_for_register(dev_priv,
2170 VLV_GTLC_PW_STATUS, mask, val,
2171 3);
ddeea5b0
ID		 2172 if (err)
2173 DRM_ERROR("timeout waiting for GT wells to go %s\n",
87ad3212 2174 onoff(wait_for_on));
ddeea5b0
ID		 2175
2176 return err;
ddeea5b0
ID		 2177}
2178
2179static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
2180{
2181 if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
2182 return;
2183
6fa283b0 2184 DRM_DEBUG_DRIVER("GT register access while GT waking disabled\n");
ddeea5b0
ID		 2185 I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
2186}
2187
ebc32824 2188static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
ddeea5b0
ID		 2189{
2190 u32 mask;
2191 int err;
2192
2193 /*
2194 * Bspec defines the following GT well on flags as debug only, so
2195 * don't treat them as hard failures.
2196 */
2197 (void)vlv_wait_for_gt_wells(dev_priv, false);
2198
2199 mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
2200 WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask);
2201
2202 vlv_check_no_gt_access(dev_priv);
2203
2204 err = vlv_force_gfx_clock(dev_priv, true);
2205 if (err)
2206 goto err1;
2207
2208 err = vlv_allow_gt_wake(dev_priv, false);
2209 if (err)
2210 goto err2;
98711167 2211
2d1fe073 2212 if (!IS_CHERRYVIEW(dev_priv))
98711167 2213 vlv_save_gunit_s0ix_state(dev_priv);
ddeea5b0
ID		 2214
2215 err = vlv_force_gfx_clock(dev_priv, false);
2216 if (err)
2217 goto err2;
2218
2219 return 0;
2220
2221err2:
2222 /* For safety always re-enable waking and disable gfx clock forcing */
2223 vlv_allow_gt_wake(dev_priv, true);
2224err1:
2225 vlv_force_gfx_clock(dev_priv, false);
2226
2227 return err;
2228}
2229
016970be
SK		 2230static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
2231 bool rpm_resume)
ddeea5b0 2232{
91c8a326 2233 struct drm_device *dev = &dev_priv->drm;
ddeea5b0
ID		 2234 int err;
2235 int ret;
2236
2237 /*
2238 * If any of the steps fail just try to continue, that's the best we
2239 * can do at this point. Return the first error code (which will also
2240 * leave RPM permanently disabled).
2241 */
2242 ret = vlv_force_gfx_clock(dev_priv, true);
2243
2d1fe073 2244 if (!IS_CHERRYVIEW(dev_priv))
98711167 2245 vlv_restore_gunit_s0ix_state(dev_priv);
ddeea5b0
ID		 2246
2247 err = vlv_allow_gt_wake(dev_priv, true);
2248 if (!ret)
2249 ret = err;
2250
2251 err = vlv_force_gfx_clock(dev_priv, false);
2252 if (!ret)
2253 ret = err;
2254
2255 vlv_check_no_gt_access(dev_priv);
2256
016970be
SK		 2257 if (rpm_resume) {
2258 intel_init_clock_gating(dev);
2259 i915_gem_restore_fences(dev);
2260 }
ddeea5b0
ID		 2261
2262 return ret;
2263}
2264
c49d13ee 2265static int intel_runtime_suspend(struct device *kdev)
8a187455 2266{
c49d13ee 2267 struct pci_dev *pdev = to_pci_dev(kdev);
8a187455 2268 struct drm_device *dev = pci_get_drvdata(pdev);
fac5e23e 2269 struct drm_i915_private *dev_priv = to_i915(dev);
0ab9cfeb 2270 int ret;
8a187455 2271
dc97997a 2272 if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6())))
c6df39b5
ID		 2273 return -ENODEV;
2274
604effb7
ID		 2275 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
2276 return -ENODEV;
2277
8a187455
PZ		 2278 DRM_DEBUG_KMS("Suspending device\n");
2279
d6102977
ID		 2280 /*
2281 * We could deadlock here in case another thread holding struct_mutex
2282 * calls RPM suspend concurrently, since the RPM suspend will wait
2283 * first for this RPM suspend to finish. In this case the concurrent
2284 * RPM resume will be followed by its RPM suspend counterpart. Still
2285 * for consistency return -EAGAIN, which will reschedule this suspend.
2286 */
2287 if (!mutex_trylock(&dev->struct_mutex)) {
2288 DRM_DEBUG_KMS("device lock contention, deffering suspend\n");
2289 /*
2290 * Bump the expiration timestamp, otherwise the suspend won't
2291 * be rescheduled.
2292 */
c49d13ee 2293 pm_runtime_mark_last_busy(kdev);
d6102977
ID		 2294
2295 return -EAGAIN;
2296 }
1f814dac
ID		 2297
2298 disable_rpm_wakeref_asserts(dev_priv);
2299
d6102977
ID		 2300 /*
2301 * We are safe here against re-faults, since the fault handler takes
2302 * an RPM reference.
2303 */
2304 i915_gem_release_all_mmaps(dev_priv);
2305 mutex_unlock(&dev->struct_mutex);
2306
a1c41994
AD		 2307 intel_guc_suspend(dev);
2308
2eb5252e 2309 intel_runtime_pm_disable_interrupts(dev_priv);
b5478bcd 2310
507e126e
ID		 2311 ret = 0;
2312 if (IS_BROXTON(dev_priv)) {
2313 bxt_display_core_uninit(dev_priv);
2314 bxt_enable_dc9(dev_priv);
2315 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2316 hsw_enable_pc8(dev_priv);
2317 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
2318 ret = vlv_suspend_complete(dev_priv);
2319 }
2320
0ab9cfeb
ID		 2321 if (ret) {
2322 DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
b963291c 2323 intel_runtime_pm_enable_interrupts(dev_priv);
0ab9cfeb 2324
1f814dac
ID		 2325 enable_rpm_wakeref_asserts(dev_priv);
2326
0ab9cfeb
ID		 2327 return ret;
2328 }
a8a8bd54 2329
dc97997a 2330 intel_uncore_forcewake_reset(dev_priv, false);
1f814dac
ID		 2331
2332 enable_rpm_wakeref_asserts(dev_priv);
2333 WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
55ec45c2 2334
bc3b9346 2335 if (intel_uncore_arm_unclaimed_mmio_detection(dev_priv))
55ec45c2
MK		 2336 DRM_ERROR("Unclaimed access detected prior to suspending\n");
2337
8a187455 2338 dev_priv->pm.suspended = true;
1fb2362b
KCA		 2339
2340 /*
c8a0bd42
PZ		 2341 * FIXME: We really should find a document that references the arguments
2342 * used below!
1fb2362b 2343 */
6f9f4b7a 2344 if (IS_BROADWELL(dev_priv)) {
d37ae19a
PZ		 2345 /*
2346 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
2347 * being detected, and the call we do at intel_runtime_resume()
2348 * won't be able to restore them. Since PCI_D3hot matches the
2349 * actual specification and appears to be working, use it.
2350 */
6f9f4b7a 2351 intel_opregion_notify_adapter(dev_priv, PCI_D3hot);
d37ae19a 2352 } else {
c8a0bd42
PZ		 2353 /*
2354 * current versions of firmware which depend on this opregion
2355 * notification have repurposed the D1 definition to mean
2356 * "runtime suspended" vs. what you would normally expect (D3)
2357 * to distinguish it from notifications that might be sent via
2358 * the suspend path.
2359 */
6f9f4b7a 2360 intel_opregion_notify_adapter(dev_priv, PCI_D1);
c8a0bd42 2361 }
8a187455 2362
59bad947 2363 assert_forcewakes_inactive(dev_priv);
dc9fb09c 2364
19625e85
L		 2365 if (!IS_VALLEYVIEW(dev_priv) || !IS_CHERRYVIEW(dev_priv))
2366 intel_hpd_poll_init(dev_priv);
2367
a8a8bd54 2368 DRM_DEBUG_KMS("Device suspended\n");
8a187455
PZ		 2369 return 0;
2370}
2371
c49d13ee 2372static int intel_runtime_resume(struct device *kdev)
8a187455 2373{
c49d13ee 2374 struct pci_dev *pdev = to_pci_dev(kdev);
8a187455 2375 struct drm_device *dev = pci_get_drvdata(pdev);
fac5e23e 2376 struct drm_i915_private *dev_priv = to_i915(dev);
1a5df187 2377 int ret = 0;
8a187455 2378
604effb7
ID		 2379 if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
2380 return -ENODEV;
8a187455
PZ		 2381
2382 DRM_DEBUG_KMS("Resuming device\n");
2383
1f814dac
ID		 2384 WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
2385 disable_rpm_wakeref_asserts(dev_priv);
2386
6f9f4b7a 2387 intel_opregion_notify_adapter(dev_priv, PCI_D0);
8a187455 2388 dev_priv->pm.suspended = false;
55ec45c2
MK		 2389 if (intel_uncore_unclaimed_mmio(dev_priv))
2390 DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n");
8a187455 2391
a1c41994
AD		 2392 intel_guc_resume(dev);
2393
1a5df187
PZ		 2394 if (IS_GEN6(dev_priv))
2395 intel_init_pch_refclk(dev);
31335cec 2396
507e126e
ID		 2397 if (IS_BROXTON(dev)) {
2398 bxt_disable_dc9(dev_priv);
2399 bxt_display_core_init(dev_priv, true);
f62c79b3
ID		 2400 if (dev_priv->csr.dmc_payload &&
2401 (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
2402 gen9_enable_dc5(dev_priv);
507e126e 2403 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
1a5df187 2404 hsw_disable_pc8(dev_priv);
507e126e 2405 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1a5df187 2406 ret = vlv_resume_prepare(dev_priv, true);
507e126e 2407 }
1a5df187 2408
0ab9cfeb
ID		 2409 /*
2410 * No point of rolling back things in case of an error, as the best
2411 * we can do is to hope that things will still work (and disable RPM).
2412 */
92b806d3 2413 i915_gem_init_swizzling(dev);
92b806d3 2414
b963291c 2415 intel_runtime_pm_enable_interrupts(dev_priv);
08d8a232
VS		 2416
2417 /*
2418 * On VLV/CHV display interrupts are part of the display
2419 * power well, so hpd is reinitialized from there. For
2420 * everyone else do it here.
2421 */
666a4537 2422 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
08d8a232
VS		 2423 intel_hpd_init(dev_priv);
2424
1f814dac
ID		 2425 enable_rpm_wakeref_asserts(dev_priv);
2426
0ab9cfeb
ID		 2427 if (ret)
2428 DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret);
2429 else
2430 DRM_DEBUG_KMS("Device resumed\n");
2431
2432 return ret;
8a187455
PZ		 2433}
2434
42f5551d 2435const struct dev_pm_ops i915_pm_ops = {
5545dbbf
ID		 2436 /*
2437 * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
2438 * PMSG_RESUME]
2439 */
0206e353 2440 .suspend = i915_pm_suspend,
76c4b250
ID		 2441 .suspend_late = i915_pm_suspend_late,
2442 .resume_early = i915_pm_resume_early,
0206e353 2443 .resume = i915_pm_resume,
5545dbbf
ID		 2444
2445 /*
2446 * S4 event handlers
2447 * @freeze, @freeze_late : called (1) before creating the
2448 * hibernation image [PMSG_FREEZE] and
2449 * (2) after rebooting, before restoring
2450 * the image [PMSG_QUIESCE]
2451 * @thaw, @thaw_early : called (1) after creating the hibernation
2452 * image, before writing it [PMSG_THAW]
2453 * and (2) after failing to create or
2454 * restore the image [PMSG_RECOVER]
2455 * @poweroff, @poweroff_late: called after writing the hibernation
2456 * image, before rebooting [PMSG_HIBERNATE]
2457 * @restore, @restore_early : called after rebooting and restoring the
2458 * hibernation image [PMSG_RESTORE]
2459 */
1f19ac2a
CW		 2460 .freeze = i915_pm_freeze,
2461 .freeze_late = i915_pm_freeze_late,
2462 .thaw_early = i915_pm_thaw_early,
2463 .thaw = i915_pm_thaw,
36d61e67 2464 .poweroff = i915_pm_suspend,
ab3be73f 2465 .poweroff_late = i915_pm_poweroff_late,
1f19ac2a
CW		 2466 .restore_early = i915_pm_restore_early,
2467 .restore = i915_pm_restore,
5545dbbf
ID		 2468
2469 /* S0ix (via runtime suspend) event handlers */
97bea207
PZ		 2470 .runtime_suspend = intel_runtime_suspend,
2471 .runtime_resume = intel_runtime_resume,
cbda12d7
ZW		 2472};
2473
78b68556 2474static const struct vm_operations_struct i915_gem_vm_ops = {
de151cf6 2475 .fault = i915_gem_fault,
ab00b3e5
JB		 2476 .open = drm_gem_vm_open,
2477 .close = drm_gem_vm_close,
de151cf6
JB		 2478};
2479
e08e96de
AV		 2480static const struct file_operations i915_driver_fops = {
2481 .owner = THIS_MODULE,
2482 .open = drm_open,
2483 .release = drm_release,
2484 .unlocked_ioctl = drm_ioctl,
2485 .mmap = drm_gem_mmap,
2486 .poll = drm_poll,
e08e96de
AV		 2487 .read = drm_read,
2488#ifdef CONFIG_COMPAT
2489 .compat_ioctl = i915_compat_ioctl,
2490#endif
2491 .llseek = noop_llseek,
2492};
2493
0673ad47
CW		 2494static int
2495i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data,
2496 struct drm_file *file)
2497{
2498 return -ENODEV;
2499}
2500
2501static const struct drm_ioctl_desc i915_ioctls[] = {
2502 DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2503 DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH),
2504 DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH),
2505 DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH),
2506 DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
2507 DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
2508 DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
2509 DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2510 DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
2511 DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
2512 DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2513 DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH),
2514 DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2515 DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2516 DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH),
2517 DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
2518 DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2519 DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2520 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH),
2521 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW),
2522 DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2523 DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2524 DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2525 DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
2526 DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
2527 DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2528 DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2529 DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2530 DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
2531 DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
2532 DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
2533 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
2534 DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW),
2535 DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
2536 DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
2537 DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW),
2538 DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW),
2539 DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
2540 DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0),
2541 DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
2542 DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
2543 DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
2544 DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW),
2545 DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW),
2546 DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2547 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
2548 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
2549 DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
2550 DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW),
2551 DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
2552 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
2553 DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
2554};
2555
1da177e4 2556static struct drm_driver driver = {
0c54781b
MW		 2557 /* Don't use MTRRs here; the Xserver or userspace app should
2558 * deal with them for Intel hardware.
792d2b9a 2559 */
673a394b 2560 .driver_features =
10ba5012 2561 DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME |
1751fcf9 2562 DRIVER_RENDER | DRIVER_MODESET,
673a394b 2563 .open = i915_driver_open,
22eae947
DA		 2564 .lastclose = i915_driver_lastclose,
2565 .preclose = i915_driver_preclose,
673a394b 2566 .postclose = i915_driver_postclose,
915b4d11 2567 .set_busid = drm_pci_set_busid,
d8e29209 2568
b1f788c6 2569 .gem_close_object = i915_gem_close_object,
673a394b 2570 .gem_free_object = i915_gem_free_object,
de151cf6 2571 .gem_vm_ops = &i915_gem_vm_ops,
1286ff73
DV		 2572
2573 .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
2574 .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
2575 .gem_prime_export = i915_gem_prime_export,
2576 .gem_prime_import = i915_gem_prime_import,
2577
ff72145b 2578 .dumb_create = i915_gem_dumb_create,
da6b51d0 2579 .dumb_map_offset = i915_gem_mmap_gtt,
43387b37 2580 .dumb_destroy = drm_gem_dumb_destroy,
1da177e4 2581 .ioctls = i915_ioctls,
0673ad47 2582 .num_ioctls = ARRAY_SIZE(i915_ioctls),
e08e96de 2583 .fops = &i915_driver_fops,
22eae947
DA		 2584 .name = DRIVER_NAME,
2585 .desc = DRIVER_DESC,
2586 .date = DRIVER_DATE,
2587 .major = DRIVER_MAJOR,
2588 .minor = DRIVER_MINOR,
2589 .patchlevel = DRIVER_PATCHLEVEL,
1da177e4 2590};
ubuntu-bionic-kernel mirror