]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/gpu/drm/i915/i915_irq.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: fix how we mask PMIMR when adding work to the queue
[mirror_ubuntu-hirsute-kernel.git] / drivers / gpu / drm / i915 / i915_irq.c
CommitLineData
0d6aa60b 1/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
1da177e4 2 */
0d6aa60b 3/*
1da177e4
LT		 4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
bc54fd1a
DA		 6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
0d6aa60b 27 */
1da177e4 28
a70491cc
JP		 29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
63eeaf38 31#include <linux/sysrq.h>
5a0e3ad6 32#include <linux/slab.h>
760285e7
DH		 33#include <drm/drmP.h>
34#include <drm/i915_drm.h>
1da177e4 35#include "i915_drv.h"
1c5d22f7 36#include "i915_trace.h"
79e53945 37#include "intel_drv.h"
1da177e4 38
e5868a31
EE		 39static const u32 hpd_ibx[] = {
40 [HPD_CRT] = SDE_CRT_HOTPLUG,
41 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
42 [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
43 [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
44 [HPD_PORT_D] = SDE_PORTD_HOTPLUG
45};
46
47static const u32 hpd_cpt[] = {
48 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
73c352a2 49 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
e5868a31
EE		 50 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
51 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
52 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
53};
54
55static const u32 hpd_mask_i915[] = {
56 [HPD_CRT] = CRT_HOTPLUG_INT_EN,
57 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
58 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
59 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
60 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
61 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
62};
63
64static const u32 hpd_status_gen4[] = {
65 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
66 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
67 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
68 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
69 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
70 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
71};
72
e5868a31
EE		 73static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
74 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
75 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
76 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
77 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
78 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
79 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
80};
81
036a4a7d 82/* For display hotplug interrupt */
995b6762 83static void
f2b115e6 84ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
036a4a7d 85{
4bc9d430
DV		 86 assert_spin_locked(&dev_priv->irq_lock);
87
1ec14ad3
CW		 88 if ((dev_priv->irq_mask & mask) != 0) {
89 dev_priv->irq_mask &= ~mask;
90 I915_WRITE(DEIMR, dev_priv->irq_mask);
3143a2bf 91 POSTING_READ(DEIMR);
036a4a7d
ZW		 92 }
93}
94
0ff9800a 95static void
f2b115e6 96ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
036a4a7d 97{
4bc9d430
DV		 98 assert_spin_locked(&dev_priv->irq_lock);
99
1ec14ad3
CW		 100 if ((dev_priv->irq_mask & mask) != mask) {
101 dev_priv->irq_mask |= mask;
102 I915_WRITE(DEIMR, dev_priv->irq_mask);
3143a2bf 103 POSTING_READ(DEIMR);
036a4a7d
ZW		 104 }
105}
106
43eaea13
PZ		 107/**
108 * ilk_update_gt_irq - update GTIMR
109 * @dev_priv: driver private
110 * @interrupt_mask: mask of interrupt bits to update
111 * @enabled_irq_mask: mask of interrupt bits to enable
112 */
113static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
114 uint32_t interrupt_mask,
115 uint32_t enabled_irq_mask)
116{
117 assert_spin_locked(&dev_priv->irq_lock);
118
119 dev_priv->gt_irq_mask &= ~interrupt_mask;
120 dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
121 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
122 POSTING_READ(GTIMR);
123}
124
125void ilk_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
126{
127 ilk_update_gt_irq(dev_priv, mask, mask);
128}
129
130void ilk_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
131{
132 ilk_update_gt_irq(dev_priv, mask, 0);
133}
134
edbfdb45
PZ		 135/**
136 * snb_update_pm_irq - update GEN6_PMIMR
137 * @dev_priv: driver private
138 * @interrupt_mask: mask of interrupt bits to update
139 * @enabled_irq_mask: mask of interrupt bits to enable
140 */
141static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
142 uint32_t interrupt_mask,
143 uint32_t enabled_irq_mask)
144{
605cd25b 145 uint32_t new_val;
edbfdb45
PZ		 146
147 assert_spin_locked(&dev_priv->irq_lock);
148
605cd25b 149 new_val = dev_priv->pm_irq_mask;
f52ecbcf
PZ		 150 new_val &= ~interrupt_mask;
151 new_val |= (~enabled_irq_mask & interrupt_mask);
152
605cd25b
PZ		 153 if (new_val != dev_priv->pm_irq_mask) {
154 dev_priv->pm_irq_mask = new_val;
155 I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
f52ecbcf
PZ		 156 POSTING_READ(GEN6_PMIMR);
157 }
edbfdb45
PZ		 158}
159
160void snb_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
161{
162 snb_update_pm_irq(dev_priv, mask, mask);
163}
164
165void snb_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
166{
167 snb_update_pm_irq(dev_priv, mask, 0);
168}
169
8664281b
PZ		 170static bool ivb_can_enable_err_int(struct drm_device *dev)
171{
172 struct drm_i915_private *dev_priv = dev->dev_private;
173 struct intel_crtc *crtc;
174 enum pipe pipe;
175
4bc9d430
DV		 176 assert_spin_locked(&dev_priv->irq_lock);
177
8664281b
PZ		 178 for_each_pipe(pipe) {
179 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
180
181 if (crtc->cpu_fifo_underrun_disabled)
182 return false;
183 }
184
185 return true;
186}
187
188static bool cpt_can_enable_serr_int(struct drm_device *dev)
189{
190 struct drm_i915_private *dev_priv = dev->dev_private;
191 enum pipe pipe;
192 struct intel_crtc *crtc;
193
fee884ed
DV		 194 assert_spin_locked(&dev_priv->irq_lock);
195
8664281b
PZ		 196 for_each_pipe(pipe) {
197 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
198
199 if (crtc->pch_fifo_underrun_disabled)
200 return false;
201 }
202
203 return true;
204}
205
206static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
207 enum pipe pipe, bool enable)
208{
209 struct drm_i915_private *dev_priv = dev->dev_private;
210 uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
211 DE_PIPEB_FIFO_UNDERRUN;
212
213 if (enable)
214 ironlake_enable_display_irq(dev_priv, bit);
215 else
216 ironlake_disable_display_irq(dev_priv, bit);
217}
218
219static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
7336df65 220 enum pipe pipe, bool enable)
8664281b
PZ		 221{
222 struct drm_i915_private *dev_priv = dev->dev_private;
8664281b 223 if (enable) {
7336df65
DV		 224 I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe));
225
8664281b
PZ		 226 if (!ivb_can_enable_err_int(dev))
227 return;
228
8664281b
PZ		 229 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
230 } else {
7336df65
DV		 231 bool was_enabled = !(I915_READ(DEIMR) & DE_ERR_INT_IVB);
232
233 /* Change the state _after_ we've read out the current one. */
8664281b 234 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
7336df65
DV		 235
236 if (!was_enabled &&
237 (I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe))) {
238 DRM_DEBUG_KMS("uncleared fifo underrun on pipe %c\n",
239 pipe_name(pipe));
240 }
8664281b
PZ		 241 }
242}
243
fee884ed
DV		 244/**
245 * ibx_display_interrupt_update - update SDEIMR
246 * @dev_priv: driver private
247 * @interrupt_mask: mask of interrupt bits to update
248 * @enabled_irq_mask: mask of interrupt bits to enable
249 */
250static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
251 uint32_t interrupt_mask,
252 uint32_t enabled_irq_mask)
253{
254 uint32_t sdeimr = I915_READ(SDEIMR);
255 sdeimr &= ~interrupt_mask;
256 sdeimr |= (~enabled_irq_mask & interrupt_mask);
257
258 assert_spin_locked(&dev_priv->irq_lock);
259
260 I915_WRITE(SDEIMR, sdeimr);
261 POSTING_READ(SDEIMR);
262}
263#define ibx_enable_display_interrupt(dev_priv, bits) \
264 ibx_display_interrupt_update((dev_priv), (bits), (bits))
265#define ibx_disable_display_interrupt(dev_priv, bits) \
266 ibx_display_interrupt_update((dev_priv), (bits), 0)
267
de28075d
DV		 268static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
269 enum transcoder pch_transcoder,
8664281b
PZ		 270 bool enable)
271{
8664281b 272 struct drm_i915_private *dev_priv = dev->dev_private;
de28075d
DV		 273 uint32_t bit = (pch_transcoder == TRANSCODER_A) ?
274 SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER;
8664281b
PZ		 275
276 if (enable)
fee884ed 277 ibx_enable_display_interrupt(dev_priv, bit);
8664281b 278 else
fee884ed 279 ibx_disable_display_interrupt(dev_priv, bit);
8664281b
PZ		 280}
281
282static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
283 enum transcoder pch_transcoder,
284 bool enable)
285{
286 struct drm_i915_private *dev_priv = dev->dev_private;
287
288 if (enable) {
1dd246fb
DV		 289 I915_WRITE(SERR_INT,
290 SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder));
291
8664281b
PZ		 292 if (!cpt_can_enable_serr_int(dev))
293 return;
294
fee884ed 295 ibx_enable_display_interrupt(dev_priv, SDE_ERROR_CPT);
8664281b 296 } else {
1dd246fb
DV		 297 uint32_t tmp = I915_READ(SERR_INT);
298 bool was_enabled = !(I915_READ(SDEIMR) & SDE_ERROR_CPT);
299
300 /* Change the state _after_ we've read out the current one. */
fee884ed 301 ibx_disable_display_interrupt(dev_priv, SDE_ERROR_CPT);
1dd246fb
DV		 302
303 if (!was_enabled &&
304 (tmp & SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder))) {
305 DRM_DEBUG_KMS("uncleared pch fifo underrun on pch transcoder %c\n",
306 transcoder_name(pch_transcoder));
307 }
8664281b 308 }
8664281b
PZ		 309}
310
311/**
312 * intel_set_cpu_fifo_underrun_reporting - enable/disable FIFO underrun messages
313 * @dev: drm device
314 * @pipe: pipe
315 * @enable: true if we want to report FIFO underrun errors, false otherwise
316 *
317 * This function makes us disable or enable CPU fifo underruns for a specific
318 * pipe. Notice that on some Gens (e.g. IVB, HSW), disabling FIFO underrun
319 * reporting for one pipe may also disable all the other CPU error interruts for
320 * the other pipes, due to the fact that there's just one interrupt mask/enable
321 * bit for all the pipes.
322 *
323 * Returns the previous state of underrun reporting.
324 */
325bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
326 enum pipe pipe, bool enable)
327{
328 struct drm_i915_private *dev_priv = dev->dev_private;
329 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
330 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
331 unsigned long flags;
332 bool ret;
333
334 spin_lock_irqsave(&dev_priv->irq_lock, flags);
335
336 ret = !intel_crtc->cpu_fifo_underrun_disabled;
337
338 if (enable == ret)
339 goto done;
340
341 intel_crtc->cpu_fifo_underrun_disabled = !enable;
342
343 if (IS_GEN5(dev) || IS_GEN6(dev))
344 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
345 else if (IS_GEN7(dev))
7336df65 346 ivybridge_set_fifo_underrun_reporting(dev, pipe, enable);
8664281b
PZ		 347
348done:
349 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
350 return ret;
351}
352
353/**
354 * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
355 * @dev: drm device
356 * @pch_transcoder: the PCH transcoder (same as pipe on IVB and older)
357 * @enable: true if we want to report FIFO underrun errors, false otherwise
358 *
359 * This function makes us disable or enable PCH fifo underruns for a specific
360 * PCH transcoder. Notice that on some PCHs (e.g. CPT/PPT), disabling FIFO
361 * underrun reporting for one transcoder may also disable all the other PCH
362 * error interruts for the other transcoders, due to the fact that there's just
363 * one interrupt mask/enable bit for all the transcoders.
364 *
365 * Returns the previous state of underrun reporting.
366 */
367bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
368 enum transcoder pch_transcoder,
369 bool enable)
370{
371 struct drm_i915_private *dev_priv = dev->dev_private;
de28075d
DV		 372 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
373 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8664281b
PZ		 374 unsigned long flags;
375 bool ret;
376
de28075d
DV		 377 /*
378 * NOTE: Pre-LPT has a fixed cpu pipe -> pch transcoder mapping, but LPT
379 * has only one pch transcoder A that all pipes can use. To avoid racy
380 * pch transcoder -> pipe lookups from interrupt code simply store the
381 * underrun statistics in crtc A. Since we never expose this anywhere
382 * nor use it outside of the fifo underrun code here using the "wrong"
383 * crtc on LPT won't cause issues.
384 */
8664281b
PZ		 385
386 spin_lock_irqsave(&dev_priv->irq_lock, flags);
387
388 ret = !intel_crtc->pch_fifo_underrun_disabled;
389
390 if (enable == ret)
391 goto done;
392
393 intel_crtc->pch_fifo_underrun_disabled = !enable;
394
395 if (HAS_PCH_IBX(dev))
de28075d 396 ibx_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
8664281b
PZ		 397 else
398 cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
399
400done:
401 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
402 return ret;
403}
404
405
7c463586
KP		 406void
407i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
408{
46c06a30
VS		 409 u32 reg = PIPESTAT(pipe);
410 u32 pipestat = I915_READ(reg) & 0x7fff0000;
7c463586 411
b79480ba
DV		 412 assert_spin_locked(&dev_priv->irq_lock);
413
46c06a30
VS		 414 if ((pipestat & mask) == mask)
415 return;
416
417 /* Enable the interrupt, clear any pending status */
418 pipestat |= mask | (mask >> 16);
419 I915_WRITE(reg, pipestat);
420 POSTING_READ(reg);
7c463586
KP		 421}
422
423void
424i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
425{
46c06a30
VS		 426 u32 reg = PIPESTAT(pipe);
427 u32 pipestat = I915_READ(reg) & 0x7fff0000;
7c463586 428
b79480ba
DV		 429 assert_spin_locked(&dev_priv->irq_lock);
430
46c06a30
VS		 431 if ((pipestat & mask) == 0)
432 return;
433
434 pipestat &= ~mask;
435 I915_WRITE(reg, pipestat);
436 POSTING_READ(reg);
7c463586
KP		 437}
438
01c66889 439/**
f49e38dd 440 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
01c66889 441 */
f49e38dd 442static void i915_enable_asle_pipestat(struct drm_device *dev)
01c66889 443{
1ec14ad3
CW		 444 drm_i915_private_t *dev_priv = dev->dev_private;
445 unsigned long irqflags;
446
f49e38dd
JN		 447 if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
448 return;
449
1ec14ad3 450 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
01c66889 451
f898780b
JN		 452 i915_enable_pipestat(dev_priv, 1, PIPE_LEGACY_BLC_EVENT_ENABLE);
453 if (INTEL_INFO(dev)->gen >= 4)
454 i915_enable_pipestat(dev_priv, 0, PIPE_LEGACY_BLC_EVENT_ENABLE);
1ec14ad3
CW		 455
456 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
01c66889
ZY		 457}
458
0a3e67a4
JB		 459/**
460 * i915_pipe_enabled - check if a pipe is enabled
461 * @dev: DRM device
462 * @pipe: pipe to check
463 *
464 * Reading certain registers when the pipe is disabled can hang the chip.
465 * Use this routine to make sure the PLL is running and the pipe is active
466 * before reading such registers if unsure.
467 */
468static int
469i915_pipe_enabled(struct drm_device *dev, int pipe)
470{
471 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
702e7a56 472
a01025af
DV		 473 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
474 /* Locking is horribly broken here, but whatever. */
475 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
476 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
71f8ba6b 477
a01025af
DV		 478 return intel_crtc->active;
479 } else {
480 return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
481 }
0a3e67a4
JB		 482}
483
42f52ef8
KP		 484/* Called from drm generic code, passed a 'crtc', which
485 * we use as a pipe index
486 */
f71d4af4 487static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
0a3e67a4
JB		 488{
489 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
490 unsigned long high_frame;
491 unsigned long low_frame;
5eddb70b 492 u32 high1, high2, low;
0a3e67a4
JB		 493
494 if (!i915_pipe_enabled(dev, pipe)) {
44d98a61 495 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
9db4a9c7 496 "pipe %c\n", pipe_name(pipe));
0a3e67a4
JB		 497 return 0;
498 }
499
9db4a9c7
JB		 500 high_frame = PIPEFRAME(pipe);
501 low_frame = PIPEFRAMEPIXEL(pipe);
5eddb70b 502
0a3e67a4
JB		 503 /*
504 * High & low register fields aren't synchronized, so make sure
505 * we get a low value that's stable across two reads of the high
506 * register.
507 */
508 do {
5eddb70b
CW		 509 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
510 low = I915_READ(low_frame) & PIPE_FRAME_LOW_MASK;
511 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
0a3e67a4
JB		 512 } while (high1 != high2);
513
5eddb70b
CW		 514 high1 >>= PIPE_FRAME_HIGH_SHIFT;
515 low >>= PIPE_FRAME_LOW_SHIFT;
516 return (high1 << 8) | low;
0a3e67a4
JB		 517}
518
f71d4af4 519static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
9880b7a5
JB		 520{
521 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
9db4a9c7 522 int reg = PIPE_FRMCOUNT_GM45(pipe);
9880b7a5
JB		 523
524 if (!i915_pipe_enabled(dev, pipe)) {
44d98a61 525 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
9db4a9c7 526 "pipe %c\n", pipe_name(pipe));
9880b7a5
JB		 527 return 0;
528 }
529
530 return I915_READ(reg);
531}
532
f71d4af4 533static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
0af7e4df
MK		 534 int *vpos, int *hpos)
535{
536 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
537 u32 vbl = 0, position = 0;
538 int vbl_start, vbl_end, htotal, vtotal;
539 bool in_vbl = true;
540 int ret = 0;
fe2b8f9d
PZ		 541 enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
542 pipe);
0af7e4df
MK		 543
544 if (!i915_pipe_enabled(dev, pipe)) {
545 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
9db4a9c7 546 "pipe %c\n", pipe_name(pipe));
0af7e4df
MK		 547 return 0;
548 }
549
550 /* Get vtotal. */
fe2b8f9d 551 vtotal = 1 + ((I915_READ(VTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
0af7e4df
MK		 552
553 if (INTEL_INFO(dev)->gen >= 4) {
554 /* No obvious pixelcount register. Only query vertical
555 * scanout position from Display scan line register.
556 */
557 position = I915_READ(PIPEDSL(pipe));
558
559 /* Decode into vertical scanout position. Don't have
560 * horizontal scanout position.
561 */
562 *vpos = position & 0x1fff;
563 *hpos = 0;
564 } else {
565 /* Have access to pixelcount since start of frame.
566 * We can split this into vertical and horizontal
567 * scanout position.
568 */
569 position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
570
fe2b8f9d 571 htotal = 1 + ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
0af7e4df
MK		 572 *vpos = position / htotal;
573 *hpos = position - (*vpos * htotal);
574 }
575
576 /* Query vblank area. */
fe2b8f9d 577 vbl = I915_READ(VBLANK(cpu_transcoder));
0af7e4df
MK		 578
579 /* Test position against vblank region. */
580 vbl_start = vbl & 0x1fff;
581 vbl_end = (vbl >> 16) & 0x1fff;
582
583 if ((*vpos < vbl_start) || (*vpos > vbl_end))
584 in_vbl = false;
585
586 /* Inside "upper part" of vblank area? Apply corrective offset: */
587 if (in_vbl && (*vpos >= vbl_start))
588 *vpos = *vpos - vtotal;
589
590 /* Readouts valid? */
591 if (vbl > 0)
592 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
593
594 /* In vblank? */
595 if (in_vbl)
596 ret |= DRM_SCANOUTPOS_INVBL;
597
598 return ret;
599}
600
f71d4af4 601static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
0af7e4df
MK		 602 int *max_error,
603 struct timeval *vblank_time,
604 unsigned flags)
605{
4041b853 606 struct drm_crtc *crtc;
0af7e4df 607
7eb552ae 608 if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
4041b853 609 DRM_ERROR("Invalid crtc %d\n", pipe);
0af7e4df
MK		 610 return -EINVAL;
611 }
612
613 /* Get drm_crtc to timestamp: */
4041b853
CW		 614 crtc = intel_get_crtc_for_pipe(dev, pipe);
615 if (crtc == NULL) {
616 DRM_ERROR("Invalid crtc %d\n", pipe);
617 return -EINVAL;
618 }
619
620 if (!crtc->enabled) {
621 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
622 return -EBUSY;
623 }
0af7e4df
MK		 624
625 /* Helper routine in DRM core does all the work: */
4041b853
CW		 626 return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
627 vblank_time, flags,
628 crtc);
0af7e4df
MK		 629}
630
321a1b30
EE		 631static int intel_hpd_irq_event(struct drm_device *dev, struct drm_connector *connector)
632{
633 enum drm_connector_status old_status;
634
635 WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
636 old_status = connector->status;
637
638 connector->status = connector->funcs->detect(connector, false);
639 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %d to %d\n",
640 connector->base.id,
641 drm_get_connector_name(connector),
642 old_status, connector->status);
643 return (old_status != connector->status);
644}
645
5ca58282
JB		 646/*
647 * Handle hotplug events outside the interrupt handler proper.
648 */
ac4c16c5
EE		 649#define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)
650
5ca58282
JB		 651static void i915_hotplug_work_func(struct work_struct *work)
652{
653 drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
654 hotplug_work);
655 struct drm_device *dev = dev_priv->dev;
c31c4ba3 656 struct drm_mode_config *mode_config = &dev->mode_config;
cd569aed
EE		 657 struct intel_connector *intel_connector;
658 struct intel_encoder *intel_encoder;
659 struct drm_connector *connector;
660 unsigned long irqflags;
661 bool hpd_disabled = false;
321a1b30 662 bool changed = false;
142e2398 663 u32 hpd_event_bits;
4ef69c7a 664
52d7eced
DV		 665 /* HPD irq before everything is fully set up. */
666 if (!dev_priv->enable_hotplug_processing)
667 return;
668
a65e34c7 669 mutex_lock(&mode_config->mutex);
e67189ab
JB		 670 DRM_DEBUG_KMS("running encoder hotplug functions\n");
671
cd569aed 672 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
142e2398
EE		 673
674 hpd_event_bits = dev_priv->hpd_event_bits;
675 dev_priv->hpd_event_bits = 0;
cd569aed
EE		 676 list_for_each_entry(connector, &mode_config->connector_list, head) {
677 intel_connector = to_intel_connector(connector);
678 intel_encoder = intel_connector->encoder;
679 if (intel_encoder->hpd_pin > HPD_NONE &&
680 dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
681 connector->polled == DRM_CONNECTOR_POLL_HPD) {
682 DRM_INFO("HPD interrupt storm detected on connector %s: "
683 "switching from hotplug detection to polling\n",
684 drm_get_connector_name(connector));
685 dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
686 connector->polled = DRM_CONNECTOR_POLL_CONNECT
687 | DRM_CONNECTOR_POLL_DISCONNECT;
688 hpd_disabled = true;
689 }
142e2398
EE		 690 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
691 DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
692 drm_get_connector_name(connector), intel_encoder->hpd_pin);
693 }
cd569aed
EE		 694 }
695 /* if there were no outputs to poll, poll was disabled,
696 * therefore make sure it's enabled when disabling HPD on
697 * some connectors */
ac4c16c5 698 if (hpd_disabled) {
cd569aed 699 drm_kms_helper_poll_enable(dev);
ac4c16c5
EE		 700 mod_timer(&dev_priv->hotplug_reenable_timer,
701 jiffies + msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
702 }
cd569aed
EE		 703
704 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
705
321a1b30
EE		 706 list_for_each_entry(connector, &mode_config->connector_list, head) {
707 intel_connector = to_intel_connector(connector);
708 intel_encoder = intel_connector->encoder;
709 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
710 if (intel_encoder->hot_plug)
711 intel_encoder->hot_plug(intel_encoder);
712 if (intel_hpd_irq_event(dev, connector))
713 changed = true;
714 }
715 }
40ee3381
KP		 716 mutex_unlock(&mode_config->mutex);
717
321a1b30
EE		 718 if (changed)
719 drm_kms_helper_hotplug_event(dev);
5ca58282
JB		 720}
721
d0ecd7e2 722static void ironlake_rps_change_irq_handler(struct drm_device *dev)
f97108d1
JB		 723{
724 drm_i915_private_t *dev_priv = dev->dev_private;
b5b72e89 725 u32 busy_up, busy_down, max_avg, min_avg;
9270388e 726 u8 new_delay;
9270388e 727
d0ecd7e2 728 spin_lock(&mchdev_lock);
f97108d1 729
73edd18f
DV		 730 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
731
20e4d407 732 new_delay = dev_priv->ips.cur_delay;
9270388e 733
7648fa99 734 I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
b5b72e89
MG		 735 busy_up = I915_READ(RCPREVBSYTUPAVG);
736 busy_down = I915_READ(RCPREVBSYTDNAVG);
f97108d1
JB		 737 max_avg = I915_READ(RCBMAXAVG);
738 min_avg = I915_READ(RCBMINAVG);
739
740 /* Handle RCS change request from hw */
b5b72e89 741 if (busy_up > max_avg) {
20e4d407
DV		 742 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
743 new_delay = dev_priv->ips.cur_delay - 1;
744 if (new_delay < dev_priv->ips.max_delay)
745 new_delay = dev_priv->ips.max_delay;
b5b72e89 746 } else if (busy_down < min_avg) {
20e4d407
DV		 747 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
748 new_delay = dev_priv->ips.cur_delay + 1;
749 if (new_delay > dev_priv->ips.min_delay)
750 new_delay = dev_priv->ips.min_delay;
f97108d1
JB		 751 }
752
7648fa99 753 if (ironlake_set_drps(dev, new_delay))
20e4d407 754 dev_priv->ips.cur_delay = new_delay;
f97108d1 755
d0ecd7e2 756 spin_unlock(&mchdev_lock);
9270388e 757
f97108d1
JB		 758 return;
759}
760
549f7365
CW		 761static void notify_ring(struct drm_device *dev,
762 struct intel_ring_buffer *ring)
763{
475553de
CW		 764 if (ring->obj == NULL)
765 return;
766
b2eadbc8 767 trace_i915_gem_request_complete(ring, ring->get_seqno(ring, false));
9862e600 768
549f7365 769 wake_up_all(&ring->irq_queue);
10cd45b6 770 i915_queue_hangcheck(dev);
549f7365
CW		 771}
772
4912d041 773static void gen6_pm_rps_work(struct work_struct *work)
3b8d8d91 774{
4912d041 775 drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
c6a828d3 776 rps.work);
edbfdb45 777 u32 pm_iir;
7b9e0ae6 778 u8 new_delay;
4912d041 779
59cdb63d 780 spin_lock_irq(&dev_priv->irq_lock);
c6a828d3
DV		 781 pm_iir = dev_priv->rps.pm_iir;
782 dev_priv->rps.pm_iir = 0;
4848405c 783 /* Make sure not to corrupt PMIMR state used by ringbuffer code */
edbfdb45 784 snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
59cdb63d 785 spin_unlock_irq(&dev_priv->irq_lock);
3b8d8d91 786
60611c13
PZ		 787 /* Make sure we didn't queue anything we're not going to process. */
788 WARN_ON(pm_iir & ~GEN6_PM_RPS_EVENTS);
789
4848405c 790 if ((pm_iir & GEN6_PM_RPS_EVENTS) == 0)
3b8d8d91
JB		 791 return;
792
4fc688ce 793 mutex_lock(&dev_priv->rps.hw_lock);
7b9e0ae6 794
7425034a 795 if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
c6a828d3 796 new_delay = dev_priv->rps.cur_delay + 1;
7425034a
VS		 797
798 /*
799 * For better performance, jump directly
800 * to RPe if we're below it.
801 */
802 if (IS_VALLEYVIEW(dev_priv->dev) &&
803 dev_priv->rps.cur_delay < dev_priv->rps.rpe_delay)
804 new_delay = dev_priv->rps.rpe_delay;
805 } else
c6a828d3 806 new_delay = dev_priv->rps.cur_delay - 1;
3b8d8d91 807
79249636
BW		 808 /* sysfs frequency interfaces may have snuck in while servicing the
809 * interrupt
810 */
d8289c9e
VS		 811 if (new_delay >= dev_priv->rps.min_delay &&
812 new_delay <= dev_priv->rps.max_delay) {
0a073b84
JB		 813 if (IS_VALLEYVIEW(dev_priv->dev))
814 valleyview_set_rps(dev_priv->dev, new_delay);
815 else
816 gen6_set_rps(dev_priv->dev, new_delay);
79249636 817 }
3b8d8d91 818
52ceb908
JB		 819 if (IS_VALLEYVIEW(dev_priv->dev)) {
820 /*
821 * On VLV, when we enter RC6 we may not be at the minimum
822 * voltage level, so arm a timer to check. It should only
823 * fire when there's activity or once after we've entered
824 * RC6, and then won't be re-armed until the next RPS interrupt.
825 */
826 mod_delayed_work(dev_priv->wq, &dev_priv->rps.vlv_work,
827 msecs_to_jiffies(100));
828 }
829
4fc688ce 830 mutex_unlock(&dev_priv->rps.hw_lock);
3b8d8d91
JB		 831}
832
e3689190
BW		 833
834/**
835 * ivybridge_parity_work - Workqueue called when a parity error interrupt
836 * occurred.
837 * @work: workqueue struct
838 *
839 * Doesn't actually do anything except notify userspace. As a consequence of
840 * this event, userspace should try to remap the bad rows since statistically
841 * it is likely the same row is more likely to go bad again.
842 */
843static void ivybridge_parity_work(struct work_struct *work)
844{
845 drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
a4da4fa4 846 l3_parity.error_work);
e3689190
BW		 847 u32 error_status, row, bank, subbank;
848 char *parity_event[5];
849 uint32_t misccpctl;
850 unsigned long flags;
851
852 /* We must turn off DOP level clock gating to access the L3 registers.
853 * In order to prevent a get/put style interface, acquire struct mutex
854 * any time we access those registers.
855 */
856 mutex_lock(&dev_priv->dev->struct_mutex);
857
858 misccpctl = I915_READ(GEN7_MISCCPCTL);
859 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
860 POSTING_READ(GEN7_MISCCPCTL);
861
862 error_status = I915_READ(GEN7_L3CDERRST1);
863 row = GEN7_PARITY_ERROR_ROW(error_status);
864 bank = GEN7_PARITY_ERROR_BANK(error_status);
865 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
866
867 I915_WRITE(GEN7_L3CDERRST1, GEN7_PARITY_ERROR_VALID |
868 GEN7_L3CDERRST1_ENABLE);
869 POSTING_READ(GEN7_L3CDERRST1);
870
871 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
872
873 spin_lock_irqsave(&dev_priv->irq_lock, flags);
43eaea13 874 ilk_enable_gt_irq(dev_priv, GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
e3689190
BW		 875 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
876
877 mutex_unlock(&dev_priv->dev->struct_mutex);
878
cce723ed 879 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
e3689190
BW		 880 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
881 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
882 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
883 parity_event[4] = NULL;
884
885 kobject_uevent_env(&dev_priv->dev->primary->kdev.kobj,
886 KOBJ_CHANGE, parity_event);
887
888 DRM_DEBUG("Parity error: Row = %d, Bank = %d, Sub bank = %d.\n",
889 row, bank, subbank);
890
891 kfree(parity_event[3]);
892 kfree(parity_event[2]);
893 kfree(parity_event[1]);
894}
895
d0ecd7e2 896static void ivybridge_parity_error_irq_handler(struct drm_device *dev)
e3689190
BW		 897{
898 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
e3689190 899
e1ef7cc2 900 if (!HAS_L3_GPU_CACHE(dev))
e3689190
BW		 901 return;
902
d0ecd7e2 903 spin_lock(&dev_priv->irq_lock);
43eaea13 904 ilk_disable_gt_irq(dev_priv, GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
d0ecd7e2 905 spin_unlock(&dev_priv->irq_lock);
e3689190 906
a4da4fa4 907 queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
e3689190
BW		 908}
909
f1af8fc1
PZ		 910static void ilk_gt_irq_handler(struct drm_device *dev,
911 struct drm_i915_private *dev_priv,
912 u32 gt_iir)
913{
914 if (gt_iir &
915 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
916 notify_ring(dev, &dev_priv->ring[RCS]);
917 if (gt_iir & ILK_BSD_USER_INTERRUPT)
918 notify_ring(dev, &dev_priv->ring[VCS]);
919}
920
e7b4c6b1
DV		 921static void snb_gt_irq_handler(struct drm_device *dev,
922 struct drm_i915_private *dev_priv,
923 u32 gt_iir)
924{
925
cc609d5d
BW		 926 if (gt_iir &
927 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
e7b4c6b1 928 notify_ring(dev, &dev_priv->ring[RCS]);
cc609d5d 929 if (gt_iir & GT_BSD_USER_INTERRUPT)
e7b4c6b1 930 notify_ring(dev, &dev_priv->ring[VCS]);
cc609d5d 931 if (gt_iir & GT_BLT_USER_INTERRUPT)
e7b4c6b1
DV		 932 notify_ring(dev, &dev_priv->ring[BCS]);
933
cc609d5d
BW		 934 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
935 GT_BSD_CS_ERROR_INTERRUPT |
936 GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
e7b4c6b1
DV		 937 DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir);
938 i915_handle_error(dev, false);
939 }
e3689190 940
cc609d5d 941 if (gt_iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
d0ecd7e2 942 ivybridge_parity_error_irq_handler(dev);
e7b4c6b1
DV		 943}
944
baf02a1f 945/* Legacy way of handling PM interrupts */
d0ecd7e2
DV		 946static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv,
947 u32 pm_iir)
fc6826d1 948{
fc6826d1
CW		 949 /*
950 * IIR bits should never already be set because IMR should
951 * prevent an interrupt from being shown in IIR. The warning
952 * displays a case where we've unsafely cleared
c6a828d3 953 * dev_priv->rps.pm_iir. Although missing an interrupt of the same
fc6826d1
CW		 954 * type is not a problem, it displays a problem in the logic.
955 *
c6a828d3 956 * The mask bit in IMR is cleared by dev_priv->rps.work.
fc6826d1
CW		 957 */
958
59cdb63d 959 spin_lock(&dev_priv->irq_lock);
60611c13 960 dev_priv->rps.pm_iir |= pm_iir & GEN6_PM_RPS_EVENTS;
4d3b3d5f 961 snb_disable_pm_irq(dev_priv, pm_iir & GEN6_PM_RPS_EVENTS);
59cdb63d 962 spin_unlock(&dev_priv->irq_lock);
fc6826d1 963
c6a828d3 964 queue_work(dev_priv->wq, &dev_priv->rps.work);
fc6826d1
CW		 965}
966
b543fb04
EE		 967#define HPD_STORM_DETECT_PERIOD 1000
968#define HPD_STORM_THRESHOLD 5
969
10a504de 970static inline void intel_hpd_irq_handler(struct drm_device *dev,
22062dba
DV		 971 u32 hotplug_trigger,
972 const u32 *hpd)
b543fb04
EE		 973{
974 drm_i915_private_t *dev_priv = dev->dev_private;
b543fb04 975 int i;
10a504de 976 bool storm_detected = false;
b543fb04 977
91d131d2
DV		 978 if (!hotplug_trigger)
979 return;
980
b5ea2d56 981 spin_lock(&dev_priv->irq_lock);
b543fb04 982 for (i = 1; i < HPD_NUM_PINS; i++) {
821450c6 983
b8f102e8
EE		 984 WARN(((hpd[i] & hotplug_trigger) &&
985 dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED),
986 "Received HPD interrupt although disabled\n");
987
b543fb04
EE		 988 if (!(hpd[i] & hotplug_trigger) ||
989 dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
990 continue;
991
bc5ead8c 992 dev_priv->hpd_event_bits |= (1 << i);
b543fb04
EE		 993 if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
994 dev_priv->hpd_stats[i].hpd_last_jiffies
995 + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
996 dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
997 dev_priv->hpd_stats[i].hpd_cnt = 0;
b8f102e8 998 DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i);
b543fb04
EE		 999 } else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
1000 dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
142e2398 1001 dev_priv->hpd_event_bits &= ~(1 << i);
b543fb04 1002 DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
10a504de 1003 storm_detected = true;
b543fb04
EE		 1004 } else {
1005 dev_priv->hpd_stats[i].hpd_cnt++;
b8f102e8
EE		 1006 DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i,
1007 dev_priv->hpd_stats[i].hpd_cnt);
b543fb04
EE		 1008 }
1009 }
1010
10a504de
DV		 1011 if (storm_detected)
1012 dev_priv->display.hpd_irq_setup(dev);
b5ea2d56 1013 spin_unlock(&dev_priv->irq_lock);
5876fa0d
DV		 1014
1015 queue_work(dev_priv->wq,
1016 &dev_priv->hotplug_work);
b543fb04
EE		 1017}
1018
515ac2bb
DV		 1019static void gmbus_irq_handler(struct drm_device *dev)
1020{
28c70f16
DV		 1021 struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
1022
28c70f16 1023 wake_up_all(&dev_priv->gmbus_wait_queue);
515ac2bb
DV		 1024}
1025
ce99c256
DV		 1026static void dp_aux_irq_handler(struct drm_device *dev)
1027{
9ee32fea
DV		 1028 struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
1029
9ee32fea 1030 wake_up_all(&dev_priv->gmbus_wait_queue);
ce99c256
DV		 1031}
1032
d0ecd7e2 1033/* Unlike gen6_rps_irq_handler() from which this function is originally derived,
baf02a1f
BW		 1034 * we must be able to deal with other PM interrupts. This is complicated because
1035 * of the way in which we use the masks to defer the RPS work (which for
1036 * posterity is necessary because of forcewake).
1037 */
1038static void hsw_pm_irq_handler(struct drm_i915_private *dev_priv,
1039 u32 pm_iir)
1040{
41a05a3a 1041 if (pm_iir & GEN6_PM_RPS_EVENTS) {
59cdb63d 1042 spin_lock(&dev_priv->irq_lock);
41a05a3a 1043 dev_priv->rps.pm_iir |= pm_iir & GEN6_PM_RPS_EVENTS;
4d3b3d5f 1044 snb_disable_pm_irq(dev_priv, pm_iir & GEN6_PM_RPS_EVENTS);
59cdb63d 1045 spin_unlock(&dev_priv->irq_lock);
2adbee62
DV		 1046
1047 queue_work(dev_priv->wq, &dev_priv->rps.work);
baf02a1f 1048 }
baf02a1f 1049
41a05a3a
DV		 1050 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1051 notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
12638c57 1052
41a05a3a
DV		 1053 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
1054 DRM_ERROR("VEBOX CS error interrupt 0x%08x\n", pm_iir);
1055 i915_handle_error(dev_priv->dev, false);
12638c57 1056 }
baf02a1f
BW		 1057}
1058
ff1f525e 1059static irqreturn_t valleyview_irq_handler(int irq, void *arg)
7e231dbe
JB		 1060{
1061 struct drm_device *dev = (struct drm_device *) arg;
1062 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1063 u32 iir, gt_iir, pm_iir;
1064 irqreturn_t ret = IRQ_NONE;
1065 unsigned long irqflags;
1066 int pipe;
1067 u32 pipe_stats[I915_MAX_PIPES];
7e231dbe
JB		 1068
1069 atomic_inc(&dev_priv->irq_received);
1070
7e231dbe
JB		 1071 while (true) {
1072 iir = I915_READ(VLV_IIR);
1073 gt_iir = I915_READ(GTIIR);
1074 pm_iir = I915_READ(GEN6_PMIIR);
1075
1076 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1077 goto out;
1078
1079 ret = IRQ_HANDLED;
1080
e7b4c6b1 1081 snb_gt_irq_handler(dev, dev_priv, gt_iir);
7e231dbe
JB		 1082
1083 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1084 for_each_pipe(pipe) {
1085 int reg = PIPESTAT(pipe);
1086 pipe_stats[pipe] = I915_READ(reg);
1087
1088 /*
1089 * Clear the PIPE*STAT regs before the IIR
1090 */
1091 if (pipe_stats[pipe] & 0x8000ffff) {
1092 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1093 DRM_DEBUG_DRIVER("pipe %c underrun\n",
1094 pipe_name(pipe));
1095 I915_WRITE(reg, pipe_stats[pipe]);
1096 }
1097 }
1098 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1099
31acc7f5
JB		 1100 for_each_pipe(pipe) {
1101 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1102 drm_handle_vblank(dev, pipe);
1103
1104 if (pipe_stats[pipe] & PLANE_FLIPDONE_INT_STATUS_VLV) {
1105 intel_prepare_page_flip(dev, pipe);
1106 intel_finish_page_flip(dev, pipe);
1107 }
1108 }
1109
7e231dbe
JB		 1110 /* Consume port. Then clear IIR or we'll miss events */
1111 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
1112 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
b543fb04 1113 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
7e231dbe
JB		 1114
1115 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
1116 hotplug_status);
91d131d2
DV		 1117
1118 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
1119
7e231dbe
JB		 1120 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1121 I915_READ(PORT_HOTPLUG_STAT);
1122 }
1123
515ac2bb
DV		 1124 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1125 gmbus_irq_handler(dev);
7e231dbe 1126
60611c13 1127 if (pm_iir)
d0ecd7e2 1128 gen6_rps_irq_handler(dev_priv, pm_iir);
7e231dbe
JB		 1129
1130 I915_WRITE(GTIIR, gt_iir);
1131 I915_WRITE(GEN6_PMIIR, pm_iir);
1132 I915_WRITE(VLV_IIR, iir);
1133 }
1134
1135out:
1136 return ret;
1137}
1138
23e81d69 1139static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
776ad806
JB		 1140{
1141 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
9db4a9c7 1142 int pipe;
b543fb04 1143 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
776ad806 1144
91d131d2
DV		 1145 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1146
cfc33bf7
VS		 1147 if (pch_iir & SDE_AUDIO_POWER_MASK) {
1148 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1149 SDE_AUDIO_POWER_SHIFT);
776ad806 1150 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
cfc33bf7
VS		 1151 port_name(port));
1152 }
776ad806 1153
ce99c256
DV		 1154 if (pch_iir & SDE_AUX_MASK)
1155 dp_aux_irq_handler(dev);
1156
776ad806 1157 if (pch_iir & SDE_GMBUS)
515ac2bb 1158 gmbus_irq_handler(dev);
776ad806
JB		 1159
1160 if (pch_iir & SDE_AUDIO_HDCP_MASK)
1161 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1162
1163 if (pch_iir & SDE_AUDIO_TRANS_MASK)
1164 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1165
1166 if (pch_iir & SDE_POISON)
1167 DRM_ERROR("PCH poison interrupt\n");
1168
9db4a9c7
JB		 1169 if (pch_iir & SDE_FDI_MASK)
1170 for_each_pipe(pipe)
1171 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1172 pipe_name(pipe),
1173 I915_READ(FDI_RX_IIR(pipe)));
776ad806
JB		 1174
1175 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1176 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1177
1178 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1179 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1180
776ad806 1181 if (pch_iir & SDE_TRANSA_FIFO_UNDER)
8664281b
PZ		 1182 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1183 false))
1184 DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1185
1186 if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1187 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1188 false))
1189 DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1190}
1191
1192static void ivb_err_int_handler(struct drm_device *dev)
1193{
1194 struct drm_i915_private *dev_priv = dev->dev_private;
1195 u32 err_int = I915_READ(GEN7_ERR_INT);
1196
de032bf4
PZ		 1197 if (err_int & ERR_INT_POISON)
1198 DRM_ERROR("Poison interrupt\n");
1199
8664281b
PZ		 1200 if (err_int & ERR_INT_FIFO_UNDERRUN_A)
1201 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1202 DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1203
1204 if (err_int & ERR_INT_FIFO_UNDERRUN_B)
1205 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1206 DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1207
1208 if (err_int & ERR_INT_FIFO_UNDERRUN_C)
1209 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_C, false))
1210 DRM_DEBUG_DRIVER("Pipe C FIFO underrun\n");
1211
1212 I915_WRITE(GEN7_ERR_INT, err_int);
1213}
1214
1215static void cpt_serr_int_handler(struct drm_device *dev)
1216{
1217 struct drm_i915_private *dev_priv = dev->dev_private;
1218 u32 serr_int = I915_READ(SERR_INT);
1219
de032bf4
PZ		 1220 if (serr_int & SERR_INT_POISON)
1221 DRM_ERROR("PCH poison interrupt\n");
1222
8664281b
PZ		 1223 if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1224 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1225 false))
1226 DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1227
1228 if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1229 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1230 false))
1231 DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1232
1233 if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1234 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
1235 false))
1236 DRM_DEBUG_DRIVER("PCH transcoder C FIFO underrun\n");
1237
1238 I915_WRITE(SERR_INT, serr_int);
776ad806
JB		 1239}
1240
23e81d69
AJ		 1241static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1242{
1243 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1244 int pipe;
b543fb04 1245 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
23e81d69 1246
91d131d2
DV		 1247 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1248
cfc33bf7
VS		 1249 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1250 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1251 SDE_AUDIO_POWER_SHIFT_CPT);
1252 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1253 port_name(port));
1254 }
23e81d69
AJ		 1255
1256 if (pch_iir & SDE_AUX_MASK_CPT)
ce99c256 1257 dp_aux_irq_handler(dev);
23e81d69
AJ		 1258
1259 if (pch_iir & SDE_GMBUS_CPT)
515ac2bb 1260 gmbus_irq_handler(dev);
23e81d69
AJ		 1261
1262 if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1263 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1264
1265 if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1266 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1267
1268 if (pch_iir & SDE_FDI_MASK_CPT)
1269 for_each_pipe(pipe)
1270 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1271 pipe_name(pipe),
1272 I915_READ(FDI_RX_IIR(pipe)));
8664281b
PZ		 1273
1274 if (pch_iir & SDE_ERROR_CPT)
1275 cpt_serr_int_handler(dev);
23e81d69
AJ		 1276}
1277
c008bc6e
PZ		 1278static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
1279{
1280 struct drm_i915_private *dev_priv = dev->dev_private;
1281
1282 if (de_iir & DE_AUX_CHANNEL_A)
1283 dp_aux_irq_handler(dev);
1284
1285 if (de_iir & DE_GSE)
1286 intel_opregion_asle_intr(dev);
1287
1288 if (de_iir & DE_PIPEA_VBLANK)
1289 drm_handle_vblank(dev, 0);
1290
1291 if (de_iir & DE_PIPEB_VBLANK)
1292 drm_handle_vblank(dev, 1);
1293
1294 if (de_iir & DE_POISON)
1295 DRM_ERROR("Poison interrupt\n");
1296
1297 if (de_iir & DE_PIPEA_FIFO_UNDERRUN)
1298 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1299 DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1300
1301 if (de_iir & DE_PIPEB_FIFO_UNDERRUN)
1302 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1303 DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1304
1305 if (de_iir & DE_PLANEA_FLIP_DONE) {
1306 intel_prepare_page_flip(dev, 0);
1307 intel_finish_page_flip_plane(dev, 0);
1308 }
1309
1310 if (de_iir & DE_PLANEB_FLIP_DONE) {
1311 intel_prepare_page_flip(dev, 1);
1312 intel_finish_page_flip_plane(dev, 1);
1313 }
1314
1315 /* check event from PCH */
1316 if (de_iir & DE_PCH_EVENT) {
1317 u32 pch_iir = I915_READ(SDEIIR);
1318
1319 if (HAS_PCH_CPT(dev))
1320 cpt_irq_handler(dev, pch_iir);
1321 else
1322 ibx_irq_handler(dev, pch_iir);
1323
1324 /* should clear PCH hotplug event before clear CPU irq */
1325 I915_WRITE(SDEIIR, pch_iir);
1326 }
1327
1328 if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
1329 ironlake_rps_change_irq_handler(dev);
1330}
1331
9719fb98
PZ		 1332static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
1333{
1334 struct drm_i915_private *dev_priv = dev->dev_private;
1335 int i;
1336
1337 if (de_iir & DE_ERR_INT_IVB)
1338 ivb_err_int_handler(dev);
1339
1340 if (de_iir & DE_AUX_CHANNEL_A_IVB)
1341 dp_aux_irq_handler(dev);
1342
1343 if (de_iir & DE_GSE_IVB)
1344 intel_opregion_asle_intr(dev);
1345
1346 for (i = 0; i < 3; i++) {
1347 if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i)))
1348 drm_handle_vblank(dev, i);
1349 if (de_iir & (DE_PLANEA_FLIP_DONE_IVB << (5 * i))) {
1350 intel_prepare_page_flip(dev, i);
1351 intel_finish_page_flip_plane(dev, i);
1352 }
1353 }
1354
1355 /* check event from PCH */
1356 if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
1357 u32 pch_iir = I915_READ(SDEIIR);
1358
1359 cpt_irq_handler(dev, pch_iir);
1360
1361 /* clear PCH hotplug event before clear CPU irq */
1362 I915_WRITE(SDEIIR, pch_iir);
1363 }
1364}
1365
f1af8fc1 1366static irqreturn_t ironlake_irq_handler(int irq, void *arg)
b1f14ad0
JB		 1367{
1368 struct drm_device *dev = (struct drm_device *) arg;
1369 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
f1af8fc1 1370 u32 de_iir, gt_iir, de_ier, sde_ier = 0;
0e43406b 1371 irqreturn_t ret = IRQ_NONE;
333a8204 1372 bool err_int_reenable = false;
b1f14ad0
JB		 1373
1374 atomic_inc(&dev_priv->irq_received);
1375
8664281b
PZ		 1376 /* We get interrupts on unclaimed registers, so check for this before we
1377 * do any I915_{READ,WRITE}. */
907b28c5 1378 intel_uncore_check_errors(dev);
8664281b 1379
b1f14ad0
JB		 1380 /* disable master interrupt before clearing iir */
1381 de_ier = I915_READ(DEIER);
1382 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
23a78516 1383 POSTING_READ(DEIER);
b1f14ad0 1384
44498aea
PZ		 1385 /* Disable south interrupts. We'll only write to SDEIIR once, so further
1386 * interrupts will will be stored on its back queue, and then we'll be
1387 * able to process them after we restore SDEIER (as soon as we restore
1388 * it, we'll get an interrupt if SDEIIR still has something to process
1389 * due to its back queue). */
ab5c608b
BW		 1390 if (!HAS_PCH_NOP(dev)) {
1391 sde_ier = I915_READ(SDEIER);
1392 I915_WRITE(SDEIER, 0);
1393 POSTING_READ(SDEIER);
1394 }
44498aea 1395
8664281b
PZ		 1396 /* On Haswell, also mask ERR_INT because we don't want to risk
1397 * generating "unclaimed register" interrupts from inside the interrupt
1398 * handler. */
4bc9d430
DV		 1399 if (IS_HASWELL(dev)) {
1400 spin_lock(&dev_priv->irq_lock);
333a8204
PZ		 1401 err_int_reenable = ~dev_priv->irq_mask & DE_ERR_INT_IVB;
1402 if (err_int_reenable)
1403 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
4bc9d430
DV		 1404 spin_unlock(&dev_priv->irq_lock);
1405 }
8664281b 1406
b1f14ad0 1407 gt_iir = I915_READ(GTIIR);
0e43406b 1408 if (gt_iir) {
d8fc8a47 1409 if (INTEL_INFO(dev)->gen >= 6)
f1af8fc1 1410 snb_gt_irq_handler(dev, dev_priv, gt_iir);
d8fc8a47
PZ		 1411 else
1412 ilk_gt_irq_handler(dev, dev_priv, gt_iir);
0e43406b
CW		 1413 I915_WRITE(GTIIR, gt_iir);
1414 ret = IRQ_HANDLED;
b1f14ad0
JB		 1415 }
1416
0e43406b
CW		 1417 de_iir = I915_READ(DEIIR);
1418 if (de_iir) {
f1af8fc1
PZ		 1419 if (INTEL_INFO(dev)->gen >= 7)
1420 ivb_display_irq_handler(dev, de_iir);
1421 else
1422 ilk_display_irq_handler(dev, de_iir);
0e43406b
CW		 1423 I915_WRITE(DEIIR, de_iir);
1424 ret = IRQ_HANDLED;
b1f14ad0
JB		 1425 }
1426
f1af8fc1
PZ		 1427 if (INTEL_INFO(dev)->gen >= 6) {
1428 u32 pm_iir = I915_READ(GEN6_PMIIR);
1429 if (pm_iir) {
1430 if (IS_HASWELL(dev))
1431 hsw_pm_irq_handler(dev_priv, pm_iir);
60611c13 1432 else
f1af8fc1
PZ		 1433 gen6_rps_irq_handler(dev_priv, pm_iir);
1434 I915_WRITE(GEN6_PMIIR, pm_iir);
1435 ret = IRQ_HANDLED;
1436 }
0e43406b 1437 }
b1f14ad0 1438
333a8204 1439 if (err_int_reenable) {
4bc9d430
DV		 1440 spin_lock(&dev_priv->irq_lock);
1441 if (ivb_can_enable_err_int(dev))
1442 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
1443 spin_unlock(&dev_priv->irq_lock);
1444 }
8664281b 1445
b1f14ad0
JB		 1446 I915_WRITE(DEIER, de_ier);
1447 POSTING_READ(DEIER);
ab5c608b
BW		 1448 if (!HAS_PCH_NOP(dev)) {
1449 I915_WRITE(SDEIER, sde_ier);
1450 POSTING_READ(SDEIER);
1451 }
b1f14ad0
JB		 1452
1453 return ret;
1454}
1455
8a905236
JB		 1456/**
1457 * i915_error_work_func - do process context error handling work
1458 * @work: work struct
1459 *
1460 * Fire an error uevent so userspace can see that a hang or error
1461 * was detected.
1462 */
1463static void i915_error_work_func(struct work_struct *work)
1464{
1f83fee0
DV		 1465 struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
1466 work);
1467 drm_i915_private_t *dev_priv = container_of(error, drm_i915_private_t,
1468 gpu_error);
8a905236 1469 struct drm_device *dev = dev_priv->dev;
f69061be 1470 struct intel_ring_buffer *ring;
cce723ed
BW		 1471 char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
1472 char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
1473 char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
f69061be 1474 int i, ret;
8a905236 1475
f316a42c
BG		 1476 kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
1477
7db0ba24
DV		 1478 /*
1479 * Note that there's only one work item which does gpu resets, so we
1480 * need not worry about concurrent gpu resets potentially incrementing
1481 * error->reset_counter twice. We only need to take care of another
1482 * racing irq/hangcheck declaring the gpu dead for a second time. A
1483 * quick check for that is good enough: schedule_work ensures the
1484 * correct ordering between hang detection and this work item, and since
1485 * the reset in-progress bit is only ever set by code outside of this
1486 * work we don't need to worry about any other races.
1487 */
1488 if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
f803aa55 1489 DRM_DEBUG_DRIVER("resetting chip\n");
7db0ba24
DV		 1490 kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE,
1491 reset_event);
1f83fee0 1492
f69061be
DV		 1493 ret = i915_reset(dev);
1494
1495 if (ret == 0) {
1496 /*
1497 * After all the gem state is reset, increment the reset
1498 * counter and wake up everyone waiting for the reset to
1499 * complete.
1500 *
1501 * Since unlock operations are a one-sided barrier only,
1502 * we need to insert a barrier here to order any seqno
1503 * updates before
1504 * the counter increment.
1505 */
1506 smp_mb__before_atomic_inc();
1507 atomic_inc(&dev_priv->gpu_error.reset_counter);
1508
1509 kobject_uevent_env(&dev->primary->kdev.kobj,
1510 KOBJ_CHANGE, reset_done_event);
1f83fee0
DV		 1511 } else {
1512 atomic_set(&error->reset_counter, I915_WEDGED);
f316a42c 1513 }
1f83fee0 1514
f69061be
DV		 1515 for_each_ring(ring, dev_priv, i)
1516 wake_up_all(&ring->irq_queue);
1517
96a02917
VS		 1518 intel_display_handle_reset(dev);
1519
1f83fee0 1520 wake_up_all(&dev_priv->gpu_error.reset_queue);
f316a42c 1521 }
8a905236
JB		 1522}
1523
35aed2e6 1524static void i915_report_and_clear_eir(struct drm_device *dev)
8a905236
JB		 1525{
1526 struct drm_i915_private *dev_priv = dev->dev_private;
bd9854f9 1527 uint32_t instdone[I915_NUM_INSTDONE_REG];
8a905236 1528 u32 eir = I915_READ(EIR);
050ee91f 1529 int pipe, i;
8a905236 1530
35aed2e6
CW		 1531 if (!eir)
1532 return;
8a905236 1533
a70491cc 1534 pr_err("render error detected, EIR: 0x%08x\n", eir);
8a905236 1535
bd9854f9
BW		 1536 i915_get_extra_instdone(dev, instdone);
1537
8a905236
JB		 1538 if (IS_G4X(dev)) {
1539 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
1540 u32 ipeir = I915_READ(IPEIR_I965);
1541
a70491cc
JP		 1542 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1543 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
050ee91f
BW		 1544 for (i = 0; i < ARRAY_SIZE(instdone); i++)
1545 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
a70491cc 1546 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
a70491cc 1547 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
8a905236 1548 I915_WRITE(IPEIR_I965, ipeir);
3143a2bf 1549 POSTING_READ(IPEIR_I965);
8a905236
JB		 1550 }
1551 if (eir & GM45_ERROR_PAGE_TABLE) {
1552 u32 pgtbl_err = I915_READ(PGTBL_ER);
a70491cc
JP		 1553 pr_err("page table error\n");
1554 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
8a905236 1555 I915_WRITE(PGTBL_ER, pgtbl_err);
3143a2bf 1556 POSTING_READ(PGTBL_ER);
8a905236
JB		 1557 }
1558 }
1559
a6c45cf0 1560 if (!IS_GEN2(dev)) {
8a905236
JB		 1561 if (eir & I915_ERROR_PAGE_TABLE) {
1562 u32 pgtbl_err = I915_READ(PGTBL_ER);
a70491cc
JP		 1563 pr_err("page table error\n");
1564 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
8a905236 1565 I915_WRITE(PGTBL_ER, pgtbl_err);
3143a2bf 1566 POSTING_READ(PGTBL_ER);
8a905236
JB		 1567 }
1568 }
1569
1570 if (eir & I915_ERROR_MEMORY_REFRESH) {
a70491cc 1571 pr_err("memory refresh error:\n");
9db4a9c7 1572 for_each_pipe(pipe)
a70491cc 1573 pr_err("pipe %c stat: 0x%08x\n",
9db4a9c7 1574 pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
8a905236
JB		 1575 /* pipestat has already been acked */
1576 }
1577 if (eir & I915_ERROR_INSTRUCTION) {
a70491cc
JP		 1578 pr_err("instruction error\n");
1579 pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM));
050ee91f
BW		 1580 for (i = 0; i < ARRAY_SIZE(instdone); i++)
1581 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
a6c45cf0 1582 if (INTEL_INFO(dev)->gen < 4) {
8a905236
JB		 1583 u32 ipeir = I915_READ(IPEIR);
1584
a70491cc
JP		 1585 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR));
1586 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR));
a70491cc 1587 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD));
8a905236 1588 I915_WRITE(IPEIR, ipeir);
3143a2bf 1589 POSTING_READ(IPEIR);
8a905236
JB		 1590 } else {
1591 u32 ipeir = I915_READ(IPEIR_I965);
1592
a70491cc
JP		 1593 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1594 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
a70491cc 1595 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
a70491cc 1596 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
8a905236 1597 I915_WRITE(IPEIR_I965, ipeir);
3143a2bf 1598 POSTING_READ(IPEIR_I965);
8a905236
JB		 1599 }
1600 }
1601
1602 I915_WRITE(EIR, eir);
3143a2bf 1603 POSTING_READ(EIR);
8a905236
JB		 1604 eir = I915_READ(EIR);
1605 if (eir) {
1606 /*
1607 * some errors might have become stuck,
1608 * mask them.
1609 */
1610 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
1611 I915_WRITE(EMR, I915_READ(EMR) | eir);
1612 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
1613 }
35aed2e6
CW		 1614}
1615
1616/**
1617 * i915_handle_error - handle an error interrupt
1618 * @dev: drm device
1619 *
1620 * Do some basic checking of regsiter state at error interrupt time and
1621 * dump it to the syslog. Also call i915_capture_error_state() to make
1622 * sure we get a record and make it available in debugfs. Fire a uevent
1623 * so userspace knows something bad happened (should trigger collection
1624 * of a ring dump etc.).
1625 */
527f9e90 1626void i915_handle_error(struct drm_device *dev, bool wedged)
35aed2e6
CW		 1627{
1628 struct drm_i915_private *dev_priv = dev->dev_private;
b4519513
CW		 1629 struct intel_ring_buffer *ring;
1630 int i;
35aed2e6
CW		 1631
1632 i915_capture_error_state(dev);
1633 i915_report_and_clear_eir(dev);
8a905236 1634
ba1234d1 1635 if (wedged) {
f69061be
DV		 1636 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
1637 &dev_priv->gpu_error.reset_counter);
ba1234d1 1638
11ed50ec 1639 /*
1f83fee0
DV		 1640 * Wakeup waiting processes so that the reset work item
1641 * doesn't deadlock trying to grab various locks.
11ed50ec 1642 */
b4519513
CW		 1643 for_each_ring(ring, dev_priv, i)
1644 wake_up_all(&ring->irq_queue);
11ed50ec
BG		 1645 }
1646
99584db3 1647 queue_work(dev_priv->wq, &dev_priv->gpu_error.work);
8a905236
JB		 1648}
1649
21ad8330 1650static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
4e5359cd
SF		 1651{
1652 drm_i915_private_t *dev_priv = dev->dev_private;
1653 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1654 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
05394f39 1655 struct drm_i915_gem_object *obj;
4e5359cd
SF		 1656 struct intel_unpin_work *work;
1657 unsigned long flags;
1658 bool stall_detected;
1659
1660 /* Ignore early vblank irqs */
1661 if (intel_crtc == NULL)
1662 return;
1663
1664 spin_lock_irqsave(&dev->event_lock, flags);
1665 work = intel_crtc->unpin_work;
1666
e7d841ca
CW		 1667 if (work == NULL ||
1668 atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
1669 !work->enable_stall_check) {
4e5359cd
SF		 1670 /* Either the pending flip IRQ arrived, or we're too early. Don't check */
1671 spin_unlock_irqrestore(&dev->event_lock, flags);
1672 return;
1673 }
1674
1675 /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
05394f39 1676 obj = work->pending_flip_obj;
a6c45cf0 1677 if (INTEL_INFO(dev)->gen >= 4) {
9db4a9c7 1678 int dspsurf = DSPSURF(intel_crtc->plane);
446f2545 1679 stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
f343c5f6 1680 i915_gem_obj_ggtt_offset(obj);
4e5359cd 1681 } else {
9db4a9c7 1682 int dspaddr = DSPADDR(intel_crtc->plane);
f343c5f6 1683 stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
01f2c773 1684 crtc->y * crtc->fb->pitches[0] +
4e5359cd
SF		 1685 crtc->x * crtc->fb->bits_per_pixel/8);
1686 }
1687
1688 spin_unlock_irqrestore(&dev->event_lock, flags);
1689
1690 if (stall_detected) {
1691 DRM_DEBUG_DRIVER("Pageflip stall detected\n");
1692 intel_prepare_page_flip(dev, intel_crtc->plane);
1693 }
1694}
1695
42f52ef8
KP		 1696/* Called from drm generic code, passed 'crtc' which
1697 * we use as a pipe index
1698 */
f71d4af4 1699static int i915_enable_vblank(struct drm_device *dev, int pipe)
0a3e67a4
JB		 1700{
1701 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
e9d21d7f 1702 unsigned long irqflags;
71e0ffa5 1703
5eddb70b 1704 if (!i915_pipe_enabled(dev, pipe))
71e0ffa5 1705 return -EINVAL;
0a3e67a4 1706
1ec14ad3 1707 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
f796cf8f 1708 if (INTEL_INFO(dev)->gen >= 4)
7c463586
KP		 1709 i915_enable_pipestat(dev_priv, pipe,
1710 PIPE_START_VBLANK_INTERRUPT_ENABLE);
e9d21d7f 1711 else
7c463586
KP		 1712 i915_enable_pipestat(dev_priv, pipe,
1713 PIPE_VBLANK_INTERRUPT_ENABLE);
8692d00e
CW		 1714
1715 /* maintain vblank delivery even in deep C-states */
1716 if (dev_priv->info->gen == 3)
6b26c86d 1717 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
1ec14ad3 1718 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
8692d00e 1719
0a3e67a4
JB		 1720 return 0;
1721}
1722
f71d4af4 1723static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
f796cf8f
JB		 1724{
1725 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1726 unsigned long irqflags;
b518421f
PZ		 1727 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
1728 DE_PIPE_VBLANK_ILK(pipe);
f796cf8f
JB		 1729
1730 if (!i915_pipe_enabled(dev, pipe))
1731 return -EINVAL;
1732
1733 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
b518421f 1734 ironlake_enable_display_irq(dev_priv, bit);
b1f14ad0
JB		 1735 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1736
1737 return 0;
1738}
1739
7e231dbe
JB		 1740static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
1741{
1742 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1743 unsigned long irqflags;
31acc7f5 1744 u32 imr;
7e231dbe
JB		 1745
1746 if (!i915_pipe_enabled(dev, pipe))
1747 return -EINVAL;
1748
1749 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
7e231dbe 1750 imr = I915_READ(VLV_IMR);
31acc7f5 1751 if (pipe == 0)
7e231dbe 1752 imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
31acc7f5 1753 else
7e231dbe 1754 imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
7e231dbe 1755 I915_WRITE(VLV_IMR, imr);
31acc7f5
JB		 1756 i915_enable_pipestat(dev_priv, pipe,
1757 PIPE_START_VBLANK_INTERRUPT_ENABLE);
7e231dbe
JB		 1758 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1759
1760 return 0;
1761}
1762
42f52ef8
KP		 1763/* Called from drm generic code, passed 'crtc' which
1764 * we use as a pipe index
1765 */
f71d4af4 1766static void i915_disable_vblank(struct drm_device *dev, int pipe)
0a3e67a4
JB		 1767{
1768 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
e9d21d7f 1769 unsigned long irqflags;
0a3e67a4 1770
1ec14ad3 1771 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
8692d00e 1772 if (dev_priv->info->gen == 3)
6b26c86d 1773 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
8692d00e 1774
f796cf8f
JB		 1775 i915_disable_pipestat(dev_priv, pipe,
1776 PIPE_VBLANK_INTERRUPT_ENABLE |
1777 PIPE_START_VBLANK_INTERRUPT_ENABLE);
1778 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1779}
1780
f71d4af4 1781static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
f796cf8f
JB		 1782{
1783 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1784 unsigned long irqflags;
b518421f
PZ		 1785 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
1786 DE_PIPE_VBLANK_ILK(pipe);
f796cf8f
JB		 1787
1788 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
b518421f 1789 ironlake_disable_display_irq(dev_priv, bit);
b1f14ad0
JB		 1790 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1791}
1792
7e231dbe
JB		 1793static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
1794{
1795 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1796 unsigned long irqflags;
31acc7f5 1797 u32 imr;
7e231dbe
JB		 1798
1799 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
31acc7f5
JB		 1800 i915_disable_pipestat(dev_priv, pipe,
1801 PIPE_START_VBLANK_INTERRUPT_ENABLE);
7e231dbe 1802 imr = I915_READ(VLV_IMR);
31acc7f5 1803 if (pipe == 0)
7e231dbe 1804 imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
31acc7f5 1805 else
7e231dbe 1806 imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
7e231dbe 1807 I915_WRITE(VLV_IMR, imr);
7e231dbe
JB		 1808 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1809}
1810
893eead0
CW		 1811static u32
1812ring_last_seqno(struct intel_ring_buffer *ring)
852835f3 1813{
893eead0
CW		 1814 return list_entry(ring->request_list.prev,
1815 struct drm_i915_gem_request, list)->seqno;
1816}
1817
9107e9d2
CW		 1818static bool
1819ring_idle(struct intel_ring_buffer *ring, u32 seqno)
1820{
1821 return (list_empty(&ring->request_list) ||
1822 i915_seqno_passed(seqno, ring_last_seqno(ring)));
f65d9421
BG		 1823}
1824
6274f212
CW		 1825static struct intel_ring_buffer *
1826semaphore_waits_for(struct intel_ring_buffer *ring, u32 *seqno)
a24a11e6
CW		 1827{
1828 struct drm_i915_private *dev_priv = ring->dev->dev_private;
6274f212 1829 u32 cmd, ipehr, acthd, acthd_min;
a24a11e6
CW		 1830
1831 ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
1832 if ((ipehr & ~(0x3 << 16)) !=
1833 (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER))
6274f212 1834 return NULL;
a24a11e6
CW		 1835
1836 /* ACTHD is likely pointing to the dword after the actual command,
1837 * so scan backwards until we find the MBOX.
1838 */
6274f212 1839 acthd = intel_ring_get_active_head(ring) & HEAD_ADDR;
a24a11e6
CW		 1840 acthd_min = max((int)acthd - 3 * 4, 0);
1841 do {
1842 cmd = ioread32(ring->virtual_start + acthd);
1843 if (cmd == ipehr)
1844 break;
1845
1846 acthd -= 4;
1847 if (acthd < acthd_min)
6274f212 1848 return NULL;
a24a11e6
CW		 1849 } while (1);
1850
6274f212
CW		 1851 *seqno = ioread32(ring->virtual_start+acthd+4)+1;
1852 return &dev_priv->ring[(ring->id + (((ipehr >> 17) & 1) + 1)) % 3];
a24a11e6
CW		 1853}
1854
6274f212
CW		 1855static int semaphore_passed(struct intel_ring_buffer *ring)
1856{
1857 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1858 struct intel_ring_buffer *signaller;
1859 u32 seqno, ctl;
1860
1861 ring->hangcheck.deadlock = true;
1862
1863 signaller = semaphore_waits_for(ring, &seqno);
1864 if (signaller == NULL || signaller->hangcheck.deadlock)
1865 return -1;
1866
1867 /* cursory check for an unkickable deadlock */
1868 ctl = I915_READ_CTL(signaller);
1869 if (ctl & RING_WAIT_SEMAPHORE && semaphore_passed(signaller) < 0)
1870 return -1;
1871
1872 return i915_seqno_passed(signaller->get_seqno(signaller, false), seqno);
1873}
1874
1875static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
1876{
1877 struct intel_ring_buffer *ring;
1878 int i;
1879
1880 for_each_ring(ring, dev_priv, i)
1881 ring->hangcheck.deadlock = false;
1882}
1883
ad8beaea
MK		 1884static enum intel_ring_hangcheck_action
1885ring_stuck(struct intel_ring_buffer *ring, u32 acthd)
1ec14ad3
CW		 1886{
1887 struct drm_device *dev = ring->dev;
1888 struct drm_i915_private *dev_priv = dev->dev_private;
9107e9d2
CW		 1889 u32 tmp;
1890
6274f212 1891 if (ring->hangcheck.acthd != acthd)
f2f4d82f 1892 return HANGCHECK_ACTIVE;
6274f212 1893
9107e9d2 1894 if (IS_GEN2(dev))
f2f4d82f 1895 return HANGCHECK_HUNG;
9107e9d2
CW		 1896
1897 /* Is the chip hanging on a WAIT_FOR_EVENT?
1898 * If so we can simply poke the RB_WAIT bit
1899 * and break the hang. This should work on
1900 * all but the second generation chipsets.
1901 */
1902 tmp = I915_READ_CTL(ring);
1ec14ad3
CW		 1903 if (tmp & RING_WAIT) {
1904 DRM_ERROR("Kicking stuck wait on %s\n",
1905 ring->name);
1906 I915_WRITE_CTL(ring, tmp);
f2f4d82f 1907 return HANGCHECK_KICK;
6274f212
CW		 1908 }
1909
1910 if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
1911 switch (semaphore_passed(ring)) {
1912 default:
f2f4d82f 1913 return HANGCHECK_HUNG;
6274f212
CW		 1914 case 1:
1915 DRM_ERROR("Kicking stuck semaphore on %s\n",
1916 ring->name);
1917 I915_WRITE_CTL(ring, tmp);
f2f4d82f 1918 return HANGCHECK_KICK;
6274f212 1919 case 0:
f2f4d82f 1920 return HANGCHECK_WAIT;
6274f212 1921 }
9107e9d2 1922 }
ed5cbb03 1923
f2f4d82f 1924 return HANGCHECK_HUNG;
ed5cbb03
MK		 1925}
1926
f65d9421
BG		 1927/**
1928 * This is called when the chip hasn't reported back with completed
05407ff8
MK		 1929 * batchbuffers in a long time. We keep track per ring seqno progress and
1930 * if there are no progress, hangcheck score for that ring is increased.
1931 * Further, acthd is inspected to see if the ring is stuck. On stuck case
1932 * we kick the ring. If we see no progress on three subsequent calls
1933 * we assume chip is wedged and try to fix it by resetting the chip.
f65d9421 1934 */
a658b5d2 1935static void i915_hangcheck_elapsed(unsigned long data)
f65d9421
BG		 1936{
1937 struct drm_device *dev = (struct drm_device *)data;
1938 drm_i915_private_t *dev_priv = dev->dev_private;
b4519513 1939 struct intel_ring_buffer *ring;
b4519513 1940 int i;
05407ff8 1941 int busy_count = 0, rings_hung = 0;
9107e9d2
CW		 1942 bool stuck[I915_NUM_RINGS] = { 0 };
1943#define BUSY 1
1944#define KICK 5
1945#define HUNG 20
1946#define FIRE 30
893eead0 1947
3e0dc6b0
BW		 1948 if (!i915_enable_hangcheck)
1949 return;
1950
b4519513 1951 for_each_ring(ring, dev_priv, i) {
05407ff8 1952 u32 seqno, acthd;
9107e9d2 1953 bool busy = true;
05407ff8 1954
6274f212
CW		 1955 semaphore_clear_deadlocks(dev_priv);
1956
05407ff8
MK		 1957 seqno = ring->get_seqno(ring, false);
1958 acthd = intel_ring_get_active_head(ring);
b4519513 1959
9107e9d2
CW		 1960 if (ring->hangcheck.seqno == seqno) {
1961 if (ring_idle(ring, seqno)) {
1962 if (waitqueue_active(&ring->irq_queue)) {
1963 /* Issue a wake-up to catch stuck h/w. */
1964 DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
1965 ring->name);
1966 wake_up_all(&ring->irq_queue);
1967 ring->hangcheck.score += HUNG;
1968 } else
1969 busy = false;
05407ff8 1970 } else {
6274f212
CW		 1971 /* We always increment the hangcheck score
1972 * if the ring is busy and still processing
1973 * the same request, so that no single request
1974 * can run indefinitely (such as a chain of
1975 * batches). The only time we do not increment
1976 * the hangcheck score on this ring, if this
1977 * ring is in a legitimate wait for another
1978 * ring. In that case the waiting ring is a
1979 * victim and we want to be sure we catch the
1980 * right culprit. Then every time we do kick
1981 * the ring, add a small increment to the
1982 * score so that we can catch a batch that is
1983 * being repeatedly kicked and so responsible
1984 * for stalling the machine.
1985 */
ad8beaea
MK		 1986 ring->hangcheck.action = ring_stuck(ring,
1987 acthd);
1988
1989 switch (ring->hangcheck.action) {
f2f4d82f 1990 case HANGCHECK_WAIT:
6274f212 1991 break;
f2f4d82f 1992 case HANGCHECK_ACTIVE:
ea04cb31 1993 ring->hangcheck.score += BUSY;
6274f212 1994 break;
f2f4d82f 1995 case HANGCHECK_KICK:
ea04cb31 1996 ring->hangcheck.score += KICK;
6274f212 1997 break;
f2f4d82f 1998 case HANGCHECK_HUNG:
ea04cb31 1999 ring->hangcheck.score += HUNG;
6274f212
CW		 2000 stuck[i] = true;
2001 break;
2002 }
05407ff8 2003 }
9107e9d2
CW		 2004 } else {
2005 /* Gradually reduce the count so that we catch DoS
2006 * attempts across multiple batches.
2007 */
2008 if (ring->hangcheck.score > 0)
2009 ring->hangcheck.score--;
d1e61e7f
CW		 2010 }
2011
05407ff8
MK		 2012 ring->hangcheck.seqno = seqno;
2013 ring->hangcheck.acthd = acthd;
9107e9d2 2014 busy_count += busy;
893eead0 2015 }
b9201c14 2016
92cab734 2017 for_each_ring(ring, dev_priv, i) {
9107e9d2 2018 if (ring->hangcheck.score > FIRE) {
acd78c11 2019 DRM_ERROR("%s on %s\n",
05407ff8 2020 stuck[i] ? "stuck" : "no progress",
a43adf07
CW		 2021 ring->name);
2022 rings_hung++;
92cab734
MK		 2023 }
2024 }
2025
05407ff8
MK		 2026 if (rings_hung)
2027 return i915_handle_error(dev, true);
f65d9421 2028
05407ff8
MK		 2029 if (busy_count)
2030 /* Reset timer case chip hangs without another request
2031 * being added */
10cd45b6
MK		 2032 i915_queue_hangcheck(dev);
2033}
2034
2035void i915_queue_hangcheck(struct drm_device *dev)
2036{
2037 struct drm_i915_private *dev_priv = dev->dev_private;
2038 if (!i915_enable_hangcheck)
2039 return;
2040
2041 mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2042 round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
f65d9421
BG		 2043}
2044
91738a95
PZ		 2045static void ibx_irq_preinstall(struct drm_device *dev)
2046{
2047 struct drm_i915_private *dev_priv = dev->dev_private;
2048
2049 if (HAS_PCH_NOP(dev))
2050 return;
2051
2052 /* south display irq */
2053 I915_WRITE(SDEIMR, 0xffffffff);
2054 /*
2055 * SDEIER is also touched by the interrupt handler to work around missed
2056 * PCH interrupts. Hence we can't update it after the interrupt handler
2057 * is enabled - instead we unconditionally enable all PCH interrupt
2058 * sources here, but then only unmask them as needed with SDEIMR.
2059 */
2060 I915_WRITE(SDEIER, 0xffffffff);
2061 POSTING_READ(SDEIER);
2062}
2063
d18ea1b5
DV		 2064static void gen5_gt_irq_preinstall(struct drm_device *dev)
2065{
2066 struct drm_i915_private *dev_priv = dev->dev_private;
2067
2068 /* and GT */
2069 I915_WRITE(GTIMR, 0xffffffff);
2070 I915_WRITE(GTIER, 0x0);
2071 POSTING_READ(GTIER);
2072
2073 if (INTEL_INFO(dev)->gen >= 6) {
2074 /* and PM */
2075 I915_WRITE(GEN6_PMIMR, 0xffffffff);
2076 I915_WRITE(GEN6_PMIER, 0x0);
2077 POSTING_READ(GEN6_PMIER);
2078 }
2079}
2080
1da177e4
LT		 2081/* drm_dma.h hooks
2082*/
f71d4af4 2083static void ironlake_irq_preinstall(struct drm_device *dev)
036a4a7d
ZW		 2084{
2085 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2086
4697995b
JB		 2087 atomic_set(&dev_priv->irq_received, 0);
2088
036a4a7d 2089 I915_WRITE(HWSTAM, 0xeffe);
bdfcdb63 2090
036a4a7d
ZW		 2091 I915_WRITE(DEIMR, 0xffffffff);
2092 I915_WRITE(DEIER, 0x0);
3143a2bf 2093 POSTING_READ(DEIER);
036a4a7d 2094
d18ea1b5 2095 gen5_gt_irq_preinstall(dev);
c650156a 2096
91738a95 2097 ibx_irq_preinstall(dev);
7d99163d
BW		 2098}
2099
7e231dbe
JB		 2100static void valleyview_irq_preinstall(struct drm_device *dev)
2101{
2102 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2103 int pipe;
2104
2105 atomic_set(&dev_priv->irq_received, 0);
2106
7e231dbe
JB		 2107 /* VLV magic */
2108 I915_WRITE(VLV_IMR, 0);
2109 I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2110 I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2111 I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2112
7e231dbe
JB		 2113 /* and GT */
2114 I915_WRITE(GTIIR, I915_READ(GTIIR));
2115 I915_WRITE(GTIIR, I915_READ(GTIIR));
d18ea1b5
DV		 2116
2117 gen5_gt_irq_preinstall(dev);
7e231dbe
JB		 2118
2119 I915_WRITE(DPINVGTT, 0xff);
2120
2121 I915_WRITE(PORT_HOTPLUG_EN, 0);
2122 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2123 for_each_pipe(pipe)
2124 I915_WRITE(PIPESTAT(pipe), 0xffff);
2125 I915_WRITE(VLV_IIR, 0xffffffff);
2126 I915_WRITE(VLV_IMR, 0xffffffff);
2127 I915_WRITE(VLV_IER, 0x0);
2128 POSTING_READ(VLV_IER);
2129}
2130
82a28bcf 2131static void ibx_hpd_irq_setup(struct drm_device *dev)
7fe0b973
KP		 2132{
2133 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
82a28bcf
DV		 2134 struct drm_mode_config *mode_config = &dev->mode_config;
2135 struct intel_encoder *intel_encoder;
fee884ed 2136 u32 hotplug_irqs, hotplug, enabled_irqs = 0;
82a28bcf
DV		 2137
2138 if (HAS_PCH_IBX(dev)) {
fee884ed 2139 hotplug_irqs = SDE_HOTPLUG_MASK;
82a28bcf 2140 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
cd569aed 2141 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
fee884ed 2142 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
82a28bcf 2143 } else {
fee884ed 2144 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
82a28bcf 2145 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
cd569aed 2146 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
fee884ed 2147 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
82a28bcf 2148 }
7fe0b973 2149
fee884ed 2150 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
82a28bcf
DV		 2151
2152 /*
2153 * Enable digital hotplug on the PCH, and configure the DP short pulse
2154 * duration to 2ms (which is the minimum in the Display Port spec)
2155 *
2156 * This register is the same on all known PCH chips.
2157 */
7fe0b973
KP		 2158 hotplug = I915_READ(PCH_PORT_HOTPLUG);
2159 hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
2160 hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
2161 hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
2162 hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
2163 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
2164}
2165
d46da437
PZ		 2166static void ibx_irq_postinstall(struct drm_device *dev)
2167{
2168 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
82a28bcf 2169 u32 mask;
e5868a31 2170
692a04cf
DV		 2171 if (HAS_PCH_NOP(dev))
2172 return;
2173
8664281b
PZ		 2174 if (HAS_PCH_IBX(dev)) {
2175 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_TRANSB_FIFO_UNDER |
de032bf4 2176 SDE_TRANSA_FIFO_UNDER | SDE_POISON;
8664281b
PZ		 2177 } else {
2178 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT | SDE_ERROR_CPT;
2179
2180 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2181 }
ab5c608b 2182
d46da437
PZ		 2183 I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2184 I915_WRITE(SDEIMR, ~mask);
d46da437
PZ		 2185}
2186
0a9a8c91
DV		 2187static void gen5_gt_irq_postinstall(struct drm_device *dev)
2188{
2189 struct drm_i915_private *dev_priv = dev->dev_private;
2190 u32 pm_irqs, gt_irqs;
2191
2192 pm_irqs = gt_irqs = 0;
2193
2194 dev_priv->gt_irq_mask = ~0;
2195 if (HAS_L3_GPU_CACHE(dev)) {
2196 /* L3 parity interrupt is always unmasked. */
2197 dev_priv->gt_irq_mask = ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2198 gt_irqs |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2199 }
2200
2201 gt_irqs |= GT_RENDER_USER_INTERRUPT;
2202 if (IS_GEN5(dev)) {
2203 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
2204 ILK_BSD_USER_INTERRUPT;
2205 } else {
2206 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
2207 }
2208
2209 I915_WRITE(GTIIR, I915_READ(GTIIR));
2210 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
2211 I915_WRITE(GTIER, gt_irqs);
2212 POSTING_READ(GTIER);
2213
2214 if (INTEL_INFO(dev)->gen >= 6) {
2215 pm_irqs |= GEN6_PM_RPS_EVENTS;
2216
2217 if (HAS_VEBOX(dev))
2218 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
2219
605cd25b 2220 dev_priv->pm_irq_mask = 0xffffffff;
0a9a8c91 2221 I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
605cd25b 2222 I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
0a9a8c91
DV		 2223 I915_WRITE(GEN6_PMIER, pm_irqs);
2224 POSTING_READ(GEN6_PMIER);
2225 }
2226}
2227
f71d4af4 2228static int ironlake_irq_postinstall(struct drm_device *dev)
036a4a7d 2229{
4bc9d430 2230 unsigned long irqflags;
036a4a7d 2231 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
8e76f8dc
PZ		 2232 u32 display_mask, extra_mask;
2233
2234 if (INTEL_INFO(dev)->gen >= 7) {
2235 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
2236 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
2237 DE_PLANEB_FLIP_DONE_IVB |
2238 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB |
2239 DE_ERR_INT_IVB);
2240 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
2241 DE_PIPEA_VBLANK_IVB);
2242
2243 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
2244 } else {
2245 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
2246 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
2247 DE_AUX_CHANNEL_A | DE_PIPEB_FIFO_UNDERRUN |
2248 DE_PIPEA_FIFO_UNDERRUN | DE_POISON);
2249 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT;
2250 }
036a4a7d 2251
1ec14ad3 2252 dev_priv->irq_mask = ~display_mask;
036a4a7d
ZW		 2253
2254 /* should always can generate irq */
2255 I915_WRITE(DEIIR, I915_READ(DEIIR));
1ec14ad3 2256 I915_WRITE(DEIMR, dev_priv->irq_mask);
8e76f8dc 2257 I915_WRITE(DEIER, display_mask | extra_mask);
3143a2bf 2258 POSTING_READ(DEIER);
036a4a7d 2259
0a9a8c91 2260 gen5_gt_irq_postinstall(dev);
036a4a7d 2261
d46da437 2262 ibx_irq_postinstall(dev);
7fe0b973 2263
f97108d1 2264 if (IS_IRONLAKE_M(dev)) {
6005ce42
DV		 2265 /* Enable PCU event interrupts
2266 *
2267 * spinlocking not required here for correctness since interrupt
4bc9d430
DV		 2268 * setup is guaranteed to run in single-threaded context. But we
2269 * need it to make the assert_spin_locked happy. */
2270 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
f97108d1 2271 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
4bc9d430 2272 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
f97108d1
JB		 2273 }
2274
036a4a7d
ZW		 2275 return 0;
2276}
2277
7e231dbe
JB		 2278static int valleyview_irq_postinstall(struct drm_device *dev)
2279{
2280 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
7e231dbe 2281 u32 enable_mask;
31acc7f5 2282 u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV;
b79480ba 2283 unsigned long irqflags;
7e231dbe
JB		 2284
2285 enable_mask = I915_DISPLAY_PORT_INTERRUPT;
31acc7f5
JB		 2286 enable_mask |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2287 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2288 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
7e231dbe
JB		 2289 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2290
31acc7f5
JB		 2291 /*
2292 *Leave vblank interrupts masked initially. enable/disable will
2293 * toggle them based on usage.
2294 */
2295 dev_priv->irq_mask = (~enable_mask) |
2296 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2297 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
7e231dbe 2298
20afbda2
DV		 2299 I915_WRITE(PORT_HOTPLUG_EN, 0);
2300 POSTING_READ(PORT_HOTPLUG_EN);
2301
7e231dbe
JB		 2302 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
2303 I915_WRITE(VLV_IER, enable_mask);
2304 I915_WRITE(VLV_IIR, 0xffffffff);
2305 I915_WRITE(PIPESTAT(0), 0xffff);
2306 I915_WRITE(PIPESTAT(1), 0xffff);
2307 POSTING_READ(VLV_IER);
2308
b79480ba
DV		 2309 /* Interrupt setup is already guaranteed to be single-threaded, this is
2310 * just to make the assert_spin_locked check happy. */
2311 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
31acc7f5 2312 i915_enable_pipestat(dev_priv, 0, pipestat_enable);
515ac2bb 2313 i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
31acc7f5 2314 i915_enable_pipestat(dev_priv, 1, pipestat_enable);
b79480ba 2315 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
31acc7f5 2316
7e231dbe
JB		 2317 I915_WRITE(VLV_IIR, 0xffffffff);
2318 I915_WRITE(VLV_IIR, 0xffffffff);
2319
0a9a8c91 2320 gen5_gt_irq_postinstall(dev);
7e231dbe
JB		 2321
2322 /* ack & enable invalid PTE error interrupts */
2323#if 0 /* FIXME: add support to irq handler for checking these bits */
2324 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2325 I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
2326#endif
2327
2328 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
20afbda2
DV		 2329
2330 return 0;
2331}
2332
7e231dbe
JB		 2333static void valleyview_irq_uninstall(struct drm_device *dev)
2334{
2335 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2336 int pipe;
2337
2338 if (!dev_priv)
2339 return;
2340
ac4c16c5
EE		 2341 del_timer_sync(&dev_priv->hotplug_reenable_timer);
2342
7e231dbe
JB		 2343 for_each_pipe(pipe)
2344 I915_WRITE(PIPESTAT(pipe), 0xffff);
2345
2346 I915_WRITE(HWSTAM, 0xffffffff);
2347 I915_WRITE(PORT_HOTPLUG_EN, 0);
2348 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2349 for_each_pipe(pipe)
2350 I915_WRITE(PIPESTAT(pipe), 0xffff);
2351 I915_WRITE(VLV_IIR, 0xffffffff);
2352 I915_WRITE(VLV_IMR, 0xffffffff);
2353 I915_WRITE(VLV_IER, 0x0);
2354 POSTING_READ(VLV_IER);
2355}
2356
f71d4af4 2357static void ironlake_irq_uninstall(struct drm_device *dev)
036a4a7d
ZW		 2358{
2359 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
4697995b
JB		 2360
2361 if (!dev_priv)
2362 return;
2363
ac4c16c5
EE		 2364 del_timer_sync(&dev_priv->hotplug_reenable_timer);
2365
036a4a7d
ZW		 2366 I915_WRITE(HWSTAM, 0xffffffff);
2367
2368 I915_WRITE(DEIMR, 0xffffffff);
2369 I915_WRITE(DEIER, 0x0);
2370 I915_WRITE(DEIIR, I915_READ(DEIIR));
8664281b
PZ		 2371 if (IS_GEN7(dev))
2372 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
036a4a7d
ZW		 2373
2374 I915_WRITE(GTIMR, 0xffffffff);
2375 I915_WRITE(GTIER, 0x0);
2376 I915_WRITE(GTIIR, I915_READ(GTIIR));
192aac1f 2377
ab5c608b
BW		 2378 if (HAS_PCH_NOP(dev))
2379 return;
2380
192aac1f
KP		 2381 I915_WRITE(SDEIMR, 0xffffffff);
2382 I915_WRITE(SDEIER, 0x0);
2383 I915_WRITE(SDEIIR, I915_READ(SDEIIR));
8664281b
PZ		 2384 if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
2385 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
036a4a7d
ZW		 2386}
2387
a266c7d5 2388static void i8xx_irq_preinstall(struct drm_device * dev)
1da177e4
LT		 2389{
2390 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
9db4a9c7 2391 int pipe;
91e3738e 2392
a266c7d5 2393 atomic_set(&dev_priv->irq_received, 0);
5ca58282 2394
9db4a9c7
JB		 2395 for_each_pipe(pipe)
2396 I915_WRITE(PIPESTAT(pipe), 0);
a266c7d5
CW		 2397 I915_WRITE16(IMR, 0xffff);
2398 I915_WRITE16(IER, 0x0);
2399 POSTING_READ16(IER);
c2798b19
CW		 2400}
2401
2402static int i8xx_irq_postinstall(struct drm_device *dev)
2403{
2404 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2405
c2798b19
CW		 2406 I915_WRITE16(EMR,
2407 ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2408
2409 /* Unmask the interrupts that we always want on. */
2410 dev_priv->irq_mask =
2411 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2412 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2413 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2414 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2415 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2416 I915_WRITE16(IMR, dev_priv->irq_mask);
2417
2418 I915_WRITE16(IER,
2419 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2420 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2421 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2422 I915_USER_INTERRUPT);
2423 POSTING_READ16(IER);
2424
2425 return 0;
2426}
2427
90a72f87
VS		 2428/*
2429 * Returns true when a page flip has completed.
2430 */
2431static bool i8xx_handle_vblank(struct drm_device *dev,
2432 int pipe, u16 iir)
2433{
2434 drm_i915_private_t *dev_priv = dev->dev_private;
2435 u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(pipe);
2436
2437 if (!drm_handle_vblank(dev, pipe))
2438 return false;
2439
2440 if ((iir & flip_pending) == 0)
2441 return false;
2442
2443 intel_prepare_page_flip(dev, pipe);
2444
2445 /* We detect FlipDone by looking for the change in PendingFlip from '1'
2446 * to '0' on the following vblank, i.e. IIR has the Pendingflip
2447 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
2448 * the flip is completed (no longer pending). Since this doesn't raise
2449 * an interrupt per se, we watch for the change at vblank.
2450 */
2451 if (I915_READ16(ISR) & flip_pending)
2452 return false;
2453
2454 intel_finish_page_flip(dev, pipe);
2455
2456 return true;
2457}
2458
ff1f525e 2459static irqreturn_t i8xx_irq_handler(int irq, void *arg)
c2798b19
CW		 2460{
2461 struct drm_device *dev = (struct drm_device *) arg;
2462 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
c2798b19
CW		 2463 u16 iir, new_iir;
2464 u32 pipe_stats[2];
2465 unsigned long irqflags;
c2798b19
CW		 2466 int pipe;
2467 u16 flip_mask =
2468 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2469 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2470
2471 atomic_inc(&dev_priv->irq_received);
2472
2473 iir = I915_READ16(IIR);
2474 if (iir == 0)
2475 return IRQ_NONE;
2476
2477 while (iir & ~flip_mask) {
2478 /* Can't rely on pipestat interrupt bit in iir as it might
2479 * have been cleared after the pipestat interrupt was received.
2480 * It doesn't set the bit in iir again, but it still produces
2481 * interrupts (for non-MSI).
2482 */
2483 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2484 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2485 i915_handle_error(dev, false);
2486
2487 for_each_pipe(pipe) {
2488 int reg = PIPESTAT(pipe);
2489 pipe_stats[pipe] = I915_READ(reg);
2490
2491 /*
2492 * Clear the PIPE*STAT regs before the IIR
2493 */
2494 if (pipe_stats[pipe] & 0x8000ffff) {
2495 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2496 DRM_DEBUG_DRIVER("pipe %c underrun\n",
2497 pipe_name(pipe));
2498 I915_WRITE(reg, pipe_stats[pipe]);
c2798b19
CW		 2499 }
2500 }
2501 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2502
2503 I915_WRITE16(IIR, iir & ~flip_mask);
2504 new_iir = I915_READ16(IIR); /* Flush posted writes */
2505
d05c617e 2506 i915_update_dri1_breadcrumb(dev);
c2798b19
CW		 2507
2508 if (iir & I915_USER_INTERRUPT)
2509 notify_ring(dev, &dev_priv->ring[RCS]);
2510
2511 if (pipe_stats[0] & PIPE_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 2512 i8xx_handle_vblank(dev, 0, iir))
2513 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(0);
c2798b19
CW		 2514
2515 if (pipe_stats[1] & PIPE_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 2516 i8xx_handle_vblank(dev, 1, iir))
2517 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(1);
c2798b19
CW		 2518
2519 iir = new_iir;
2520 }
2521
2522 return IRQ_HANDLED;
2523}
2524
2525static void i8xx_irq_uninstall(struct drm_device * dev)
2526{
2527 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2528 int pipe;
2529
c2798b19
CW		 2530 for_each_pipe(pipe) {
2531 /* Clear enable bits; then clear status bits */
2532 I915_WRITE(PIPESTAT(pipe), 0);
2533 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2534 }
2535 I915_WRITE16(IMR, 0xffff);
2536 I915_WRITE16(IER, 0x0);
2537 I915_WRITE16(IIR, I915_READ16(IIR));
2538}
2539
a266c7d5
CW		 2540static void i915_irq_preinstall(struct drm_device * dev)
2541{
2542 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2543 int pipe;
2544
2545 atomic_set(&dev_priv->irq_received, 0);
2546
2547 if (I915_HAS_HOTPLUG(dev)) {
2548 I915_WRITE(PORT_HOTPLUG_EN, 0);
2549 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2550 }
2551
00d98ebd 2552 I915_WRITE16(HWSTAM, 0xeffe);
a266c7d5
CW		 2553 for_each_pipe(pipe)
2554 I915_WRITE(PIPESTAT(pipe), 0);
2555 I915_WRITE(IMR, 0xffffffff);
2556 I915_WRITE(IER, 0x0);
2557 POSTING_READ(IER);
2558}
2559
2560static int i915_irq_postinstall(struct drm_device *dev)
2561{
2562 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
38bde180 2563 u32 enable_mask;
a266c7d5 2564
38bde180
CW		 2565 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2566
2567 /* Unmask the interrupts that we always want on. */
2568 dev_priv->irq_mask =
2569 ~(I915_ASLE_INTERRUPT |
2570 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2571 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2572 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2573 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2574 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2575
2576 enable_mask =
2577 I915_ASLE_INTERRUPT |
2578 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2579 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2580 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2581 I915_USER_INTERRUPT;
2582
a266c7d5 2583 if (I915_HAS_HOTPLUG(dev)) {
20afbda2
DV		 2584 I915_WRITE(PORT_HOTPLUG_EN, 0);
2585 POSTING_READ(PORT_HOTPLUG_EN);
2586
a266c7d5
CW		 2587 /* Enable in IER... */
2588 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
2589 /* and unmask in IMR */
2590 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
2591 }
2592
a266c7d5
CW		 2593 I915_WRITE(IMR, dev_priv->irq_mask);
2594 I915_WRITE(IER, enable_mask);
2595 POSTING_READ(IER);
2596
f49e38dd 2597 i915_enable_asle_pipestat(dev);
20afbda2
DV		 2598
2599 return 0;
2600}
2601
90a72f87
VS		 2602/*
2603 * Returns true when a page flip has completed.
2604 */
2605static bool i915_handle_vblank(struct drm_device *dev,
2606 int plane, int pipe, u32 iir)
2607{
2608 drm_i915_private_t *dev_priv = dev->dev_private;
2609 u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
2610
2611 if (!drm_handle_vblank(dev, pipe))
2612 return false;
2613
2614 if ((iir & flip_pending) == 0)
2615 return false;
2616
2617 intel_prepare_page_flip(dev, plane);
2618
2619 /* We detect FlipDone by looking for the change in PendingFlip from '1'
2620 * to '0' on the following vblank, i.e. IIR has the Pendingflip
2621 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
2622 * the flip is completed (no longer pending). Since this doesn't raise
2623 * an interrupt per se, we watch for the change at vblank.
2624 */
2625 if (I915_READ(ISR) & flip_pending)
2626 return false;
2627
2628 intel_finish_page_flip(dev, pipe);
2629
2630 return true;
2631}
2632
ff1f525e 2633static irqreturn_t i915_irq_handler(int irq, void *arg)
a266c7d5
CW		 2634{
2635 struct drm_device *dev = (struct drm_device *) arg;
2636 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
8291ee90 2637 u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
a266c7d5 2638 unsigned long irqflags;
38bde180
CW		 2639 u32 flip_mask =
2640 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2641 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
38bde180 2642 int pipe, ret = IRQ_NONE;
a266c7d5
CW		 2643
2644 atomic_inc(&dev_priv->irq_received);
2645
2646 iir = I915_READ(IIR);
38bde180
CW		 2647 do {
2648 bool irq_received = (iir & ~flip_mask) != 0;
8291ee90 2649 bool blc_event = false;
a266c7d5
CW		 2650
2651 /* Can't rely on pipestat interrupt bit in iir as it might
2652 * have been cleared after the pipestat interrupt was received.
2653 * It doesn't set the bit in iir again, but it still produces
2654 * interrupts (for non-MSI).
2655 */
2656 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2657 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2658 i915_handle_error(dev, false);
2659
2660 for_each_pipe(pipe) {
2661 int reg = PIPESTAT(pipe);
2662 pipe_stats[pipe] = I915_READ(reg);
2663
38bde180 2664 /* Clear the PIPE*STAT regs before the IIR */
a266c7d5
CW		 2665 if (pipe_stats[pipe] & 0x8000ffff) {
2666 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2667 DRM_DEBUG_DRIVER("pipe %c underrun\n",
2668 pipe_name(pipe));
2669 I915_WRITE(reg, pipe_stats[pipe]);
38bde180 2670 irq_received = true;
a266c7d5
CW		 2671 }
2672 }
2673 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2674
2675 if (!irq_received)
2676 break;
2677
a266c7d5
CW		 2678 /* Consume port. Then clear IIR or we'll miss events */
2679 if ((I915_HAS_HOTPLUG(dev)) &&
2680 (iir & I915_DISPLAY_PORT_INTERRUPT)) {
2681 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
b543fb04 2682 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
a266c7d5
CW		 2683
2684 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2685 hotplug_status);
91d131d2
DV		 2686
2687 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
2688
a266c7d5 2689 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
38bde180 2690 POSTING_READ(PORT_HOTPLUG_STAT);
a266c7d5
CW		 2691 }
2692
38bde180 2693 I915_WRITE(IIR, iir & ~flip_mask);
a266c7d5
CW		 2694 new_iir = I915_READ(IIR); /* Flush posted writes */
2695
a266c7d5
CW		 2696 if (iir & I915_USER_INTERRUPT)
2697 notify_ring(dev, &dev_priv->ring[RCS]);
a266c7d5 2698
a266c7d5 2699 for_each_pipe(pipe) {
38bde180
CW		 2700 int plane = pipe;
2701 if (IS_MOBILE(dev))
2702 plane = !plane;
90a72f87 2703
8291ee90 2704 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 2705 i915_handle_vblank(dev, plane, pipe, iir))
2706 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
a266c7d5
CW		 2707
2708 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2709 blc_event = true;
2710 }
2711
a266c7d5
CW		 2712 if (blc_event || (iir & I915_ASLE_INTERRUPT))
2713 intel_opregion_asle_intr(dev);
2714
2715 /* With MSI, interrupts are only generated when iir
2716 * transitions from zero to nonzero. If another bit got
2717 * set while we were handling the existing iir bits, then
2718 * we would never get another interrupt.
2719 *
2720 * This is fine on non-MSI as well, as if we hit this path
2721 * we avoid exiting the interrupt handler only to generate
2722 * another one.
2723 *
2724 * Note that for MSI this could cause a stray interrupt report
2725 * if an interrupt landed in the time between writing IIR and
2726 * the posting read. This should be rare enough to never
2727 * trigger the 99% of 100,000 interrupts test for disabling
2728 * stray interrupts.
2729 */
38bde180 2730 ret = IRQ_HANDLED;
a266c7d5 2731 iir = new_iir;
38bde180 2732 } while (iir & ~flip_mask);
a266c7d5 2733
d05c617e 2734 i915_update_dri1_breadcrumb(dev);
8291ee90 2735
a266c7d5
CW		 2736 return ret;
2737}
2738
2739static void i915_irq_uninstall(struct drm_device * dev)
2740{
2741 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2742 int pipe;
2743
ac4c16c5
EE		 2744 del_timer_sync(&dev_priv->hotplug_reenable_timer);
2745
a266c7d5
CW		 2746 if (I915_HAS_HOTPLUG(dev)) {
2747 I915_WRITE(PORT_HOTPLUG_EN, 0);
2748 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2749 }
2750
00d98ebd 2751 I915_WRITE16(HWSTAM, 0xffff);
55b39755
CW		 2752 for_each_pipe(pipe) {
2753 /* Clear enable bits; then clear status bits */
a266c7d5 2754 I915_WRITE(PIPESTAT(pipe), 0);
55b39755
CW		 2755 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2756 }
a266c7d5
CW		 2757 I915_WRITE(IMR, 0xffffffff);
2758 I915_WRITE(IER, 0x0);
2759
a266c7d5
CW		 2760 I915_WRITE(IIR, I915_READ(IIR));
2761}
2762
2763static void i965_irq_preinstall(struct drm_device * dev)
2764{
2765 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2766 int pipe;
2767
2768 atomic_set(&dev_priv->irq_received, 0);
2769
adca4730
CW		 2770 I915_WRITE(PORT_HOTPLUG_EN, 0);
2771 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
a266c7d5
CW		 2772
2773 I915_WRITE(HWSTAM, 0xeffe);
2774 for_each_pipe(pipe)
2775 I915_WRITE(PIPESTAT(pipe), 0);
2776 I915_WRITE(IMR, 0xffffffff);
2777 I915_WRITE(IER, 0x0);
2778 POSTING_READ(IER);
2779}
2780
2781static int i965_irq_postinstall(struct drm_device *dev)
2782{
2783 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
bbba0a97 2784 u32 enable_mask;
a266c7d5 2785 u32 error_mask;
b79480ba 2786 unsigned long irqflags;
a266c7d5 2787
a266c7d5 2788 /* Unmask the interrupts that we always want on. */
bbba0a97 2789 dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
adca4730 2790 I915_DISPLAY_PORT_INTERRUPT |
bbba0a97
CW		 2791 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2792 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2793 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2794 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2795 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2796
2797 enable_mask = ~dev_priv->irq_mask;
21ad8330
VS		 2798 enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2799 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
bbba0a97
CW		 2800 enable_mask |= I915_USER_INTERRUPT;
2801
2802 if (IS_G4X(dev))
2803 enable_mask |= I915_BSD_USER_INTERRUPT;
a266c7d5 2804
b79480ba
DV		 2805 /* Interrupt setup is already guaranteed to be single-threaded, this is
2806 * just to make the assert_spin_locked check happy. */
2807 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
515ac2bb 2808 i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
b79480ba 2809 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
a266c7d5 2810
a266c7d5
CW		 2811 /*
2812 * Enable some error detection, note the instruction error mask
2813 * bit is reserved, so we leave it masked.
2814 */
2815 if (IS_G4X(dev)) {
2816 error_mask = ~(GM45_ERROR_PAGE_TABLE |
2817 GM45_ERROR_MEM_PRIV |
2818 GM45_ERROR_CP_PRIV |
2819 I915_ERROR_MEMORY_REFRESH);
2820 } else {
2821 error_mask = ~(I915_ERROR_PAGE_TABLE |
2822 I915_ERROR_MEMORY_REFRESH);
2823 }
2824 I915_WRITE(EMR, error_mask);
2825
2826 I915_WRITE(IMR, dev_priv->irq_mask);
2827 I915_WRITE(IER, enable_mask);
2828 POSTING_READ(IER);
2829
20afbda2
DV		 2830 I915_WRITE(PORT_HOTPLUG_EN, 0);
2831 POSTING_READ(PORT_HOTPLUG_EN);
2832
f49e38dd 2833 i915_enable_asle_pipestat(dev);
20afbda2
DV		 2834
2835 return 0;
2836}
2837
bac56d5b 2838static void i915_hpd_irq_setup(struct drm_device *dev)
20afbda2
DV		 2839{
2840 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
e5868a31 2841 struct drm_mode_config *mode_config = &dev->mode_config;
cd569aed 2842 struct intel_encoder *intel_encoder;
20afbda2
DV		 2843 u32 hotplug_en;
2844
b5ea2d56
DV		 2845 assert_spin_locked(&dev_priv->irq_lock);
2846
bac56d5b
EE		 2847 if (I915_HAS_HOTPLUG(dev)) {
2848 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
2849 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
2850 /* Note HDMI and DP share hotplug bits */
e5868a31 2851 /* enable bits are the same for all generations */
cd569aed
EE		 2852 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2853 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2854 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
bac56d5b
EE		 2855 /* Programming the CRT detection parameters tends
2856 to generate a spurious hotplug event about three
2857 seconds later. So just do it once.
2858 */
2859 if (IS_G4X(dev))
2860 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
85fc95ba 2861 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
bac56d5b 2862 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
a266c7d5 2863
bac56d5b
EE		 2864 /* Ignore TV since it's buggy */
2865 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
2866 }
a266c7d5
CW		 2867}
2868
ff1f525e 2869static irqreturn_t i965_irq_handler(int irq, void *arg)
a266c7d5
CW		 2870{
2871 struct drm_device *dev = (struct drm_device *) arg;
2872 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
a266c7d5
CW		 2873 u32 iir, new_iir;
2874 u32 pipe_stats[I915_MAX_PIPES];
a266c7d5
CW		 2875 unsigned long irqflags;
2876 int irq_received;
2877 int ret = IRQ_NONE, pipe;
21ad8330
VS		 2878 u32 flip_mask =
2879 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2880 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
a266c7d5
CW		 2881
2882 atomic_inc(&dev_priv->irq_received);
2883
2884 iir = I915_READ(IIR);
2885
a266c7d5 2886 for (;;) {
2c8ba29f
CW		 2887 bool blc_event = false;
2888
21ad8330 2889 irq_received = (iir & ~flip_mask) != 0;
a266c7d5
CW		 2890
2891 /* Can't rely on pipestat interrupt bit in iir as it might
2892 * have been cleared after the pipestat interrupt was received.
2893 * It doesn't set the bit in iir again, but it still produces
2894 * interrupts (for non-MSI).
2895 */
2896 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2897 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2898 i915_handle_error(dev, false);
2899
2900 for_each_pipe(pipe) {
2901 int reg = PIPESTAT(pipe);
2902 pipe_stats[pipe] = I915_READ(reg);
2903
2904 /*
2905 * Clear the PIPE*STAT regs before the IIR
2906 */
2907 if (pipe_stats[pipe] & 0x8000ffff) {
2908 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2909 DRM_DEBUG_DRIVER("pipe %c underrun\n",
2910 pipe_name(pipe));
2911 I915_WRITE(reg, pipe_stats[pipe]);
2912 irq_received = 1;
2913 }
2914 }
2915 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2916
2917 if (!irq_received)
2918 break;
2919
2920 ret = IRQ_HANDLED;
2921
2922 /* Consume port. Then clear IIR or we'll miss events */
adca4730 2923 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
a266c7d5 2924 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
b543fb04
EE		 2925 u32 hotplug_trigger = hotplug_status & (IS_G4X(dev) ?
2926 HOTPLUG_INT_STATUS_G4X :
4f7fd709 2927 HOTPLUG_INT_STATUS_I915);
a266c7d5
CW		 2928
2929 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2930 hotplug_status);
91d131d2
DV		 2931
2932 intel_hpd_irq_handler(dev, hotplug_trigger,
2933 IS_G4X(dev) ? hpd_status_gen4 : hpd_status_i915);
2934
a266c7d5
CW		 2935 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2936 I915_READ(PORT_HOTPLUG_STAT);
2937 }
2938
21ad8330 2939 I915_WRITE(IIR, iir & ~flip_mask);
a266c7d5
CW		 2940 new_iir = I915_READ(IIR); /* Flush posted writes */
2941
a266c7d5
CW		 2942 if (iir & I915_USER_INTERRUPT)
2943 notify_ring(dev, &dev_priv->ring[RCS]);
2944 if (iir & I915_BSD_USER_INTERRUPT)
2945 notify_ring(dev, &dev_priv->ring[VCS]);
2946
a266c7d5 2947 for_each_pipe(pipe) {
2c8ba29f 2948 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 2949 i915_handle_vblank(dev, pipe, pipe, iir))
2950 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
a266c7d5
CW		 2951
2952 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2953 blc_event = true;
2954 }
2955
2956
2957 if (blc_event || (iir & I915_ASLE_INTERRUPT))
2958 intel_opregion_asle_intr(dev);
2959
515ac2bb
DV		 2960 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
2961 gmbus_irq_handler(dev);
2962
a266c7d5
CW		 2963 /* With MSI, interrupts are only generated when iir
2964 * transitions from zero to nonzero. If another bit got
2965 * set while we were handling the existing iir bits, then
2966 * we would never get another interrupt.
2967 *
2968 * This is fine on non-MSI as well, as if we hit this path
2969 * we avoid exiting the interrupt handler only to generate
2970 * another one.
2971 *
2972 * Note that for MSI this could cause a stray interrupt report
2973 * if an interrupt landed in the time between writing IIR and
2974 * the posting read. This should be rare enough to never
2975 * trigger the 99% of 100,000 interrupts test for disabling
2976 * stray interrupts.
2977 */
2978 iir = new_iir;
2979 }
2980
d05c617e 2981 i915_update_dri1_breadcrumb(dev);
2c8ba29f 2982
a266c7d5
CW		 2983 return ret;
2984}
2985
2986static void i965_irq_uninstall(struct drm_device * dev)
2987{
2988 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2989 int pipe;
2990
2991 if (!dev_priv)
2992 return;
2993
ac4c16c5
EE		 2994 del_timer_sync(&dev_priv->hotplug_reenable_timer);
2995
adca4730
CW		 2996 I915_WRITE(PORT_HOTPLUG_EN, 0);
2997 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
a266c7d5
CW		 2998
2999 I915_WRITE(HWSTAM, 0xffffffff);
3000 for_each_pipe(pipe)
3001 I915_WRITE(PIPESTAT(pipe), 0);
3002 I915_WRITE(IMR, 0xffffffff);
3003 I915_WRITE(IER, 0x0);
3004
3005 for_each_pipe(pipe)
3006 I915_WRITE(PIPESTAT(pipe),
3007 I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
3008 I915_WRITE(IIR, I915_READ(IIR));
3009}
3010
ac4c16c5
EE		 3011static void i915_reenable_hotplug_timer_func(unsigned long data)
3012{
3013 drm_i915_private_t *dev_priv = (drm_i915_private_t *)data;
3014 struct drm_device *dev = dev_priv->dev;
3015 struct drm_mode_config *mode_config = &dev->mode_config;
3016 unsigned long irqflags;
3017 int i;
3018
3019 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3020 for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
3021 struct drm_connector *connector;
3022
3023 if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
3024 continue;
3025
3026 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3027
3028 list_for_each_entry(connector, &mode_config->connector_list, head) {
3029 struct intel_connector *intel_connector = to_intel_connector(connector);
3030
3031 if (intel_connector->encoder->hpd_pin == i) {
3032 if (connector->polled != intel_connector->polled)
3033 DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
3034 drm_get_connector_name(connector));
3035 connector->polled = intel_connector->polled;
3036 if (!connector->polled)
3037 connector->polled = DRM_CONNECTOR_POLL_HPD;
3038 }
3039 }
3040 }
3041 if (dev_priv->display.hpd_irq_setup)
3042 dev_priv->display.hpd_irq_setup(dev);
3043 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3044}
3045
f71d4af4
JB		 3046void intel_irq_init(struct drm_device *dev)
3047{
8b2e326d
CW		 3048 struct drm_i915_private *dev_priv = dev->dev_private;
3049
3050 INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
99584db3 3051 INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
c6a828d3 3052 INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
a4da4fa4 3053 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
8b2e326d 3054
99584db3
DV		 3055 setup_timer(&dev_priv->gpu_error.hangcheck_timer,
3056 i915_hangcheck_elapsed,
61bac78e 3057 (unsigned long) dev);
ac4c16c5
EE		 3058 setup_timer(&dev_priv->hotplug_reenable_timer, i915_reenable_hotplug_timer_func,
3059 (unsigned long) dev_priv);
61bac78e 3060
97a19a24 3061 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
9ee32fea 3062
f71d4af4
JB		 3063 dev->driver->get_vblank_counter = i915_get_vblank_counter;
3064 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
7d4e146f 3065 if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
f71d4af4
JB		 3066 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
3067 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
3068 }
3069
c3613de9
KP		 3070 if (drm_core_check_feature(dev, DRIVER_MODESET))
3071 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
3072 else
3073 dev->driver->get_vblank_timestamp = NULL;
f71d4af4
JB		 3074 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
3075
7e231dbe
JB		 3076 if (IS_VALLEYVIEW(dev)) {
3077 dev->driver->irq_handler = valleyview_irq_handler;
3078 dev->driver->irq_preinstall = valleyview_irq_preinstall;
3079 dev->driver->irq_postinstall = valleyview_irq_postinstall;
3080 dev->driver->irq_uninstall = valleyview_irq_uninstall;
3081 dev->driver->enable_vblank = valleyview_enable_vblank;
3082 dev->driver->disable_vblank = valleyview_disable_vblank;
fa00abe0 3083 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
f71d4af4
JB		 3084 } else if (HAS_PCH_SPLIT(dev)) {
3085 dev->driver->irq_handler = ironlake_irq_handler;
3086 dev->driver->irq_preinstall = ironlake_irq_preinstall;
3087 dev->driver->irq_postinstall = ironlake_irq_postinstall;
3088 dev->driver->irq_uninstall = ironlake_irq_uninstall;
3089 dev->driver->enable_vblank = ironlake_enable_vblank;
3090 dev->driver->disable_vblank = ironlake_disable_vblank;
82a28bcf 3091 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
f71d4af4 3092 } else {
c2798b19
CW		 3093 if (INTEL_INFO(dev)->gen == 2) {
3094 dev->driver->irq_preinstall = i8xx_irq_preinstall;
3095 dev->driver->irq_postinstall = i8xx_irq_postinstall;
3096 dev->driver->irq_handler = i8xx_irq_handler;
3097 dev->driver->irq_uninstall = i8xx_irq_uninstall;
a266c7d5
CW		 3098 } else if (INTEL_INFO(dev)->gen == 3) {
3099 dev->driver->irq_preinstall = i915_irq_preinstall;
3100 dev->driver->irq_postinstall = i915_irq_postinstall;
3101 dev->driver->irq_uninstall = i915_irq_uninstall;
3102 dev->driver->irq_handler = i915_irq_handler;
20afbda2 3103 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
c2798b19 3104 } else {
a266c7d5
CW		 3105 dev->driver->irq_preinstall = i965_irq_preinstall;
3106 dev->driver->irq_postinstall = i965_irq_postinstall;
3107 dev->driver->irq_uninstall = i965_irq_uninstall;
3108 dev->driver->irq_handler = i965_irq_handler;
bac56d5b 3109 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
c2798b19 3110 }
f71d4af4
JB		 3111 dev->driver->enable_vblank = i915_enable_vblank;
3112 dev->driver->disable_vblank = i915_disable_vblank;
3113 }
3114}
20afbda2
DV		 3115
3116void intel_hpd_init(struct drm_device *dev)
3117{
3118 struct drm_i915_private *dev_priv = dev->dev_private;
821450c6
EE		 3119 struct drm_mode_config *mode_config = &dev->mode_config;
3120 struct drm_connector *connector;
b5ea2d56 3121 unsigned long irqflags;
821450c6 3122 int i;
20afbda2 3123
821450c6
EE		 3124 for (i = 1; i < HPD_NUM_PINS; i++) {
3125 dev_priv->hpd_stats[i].hpd_cnt = 0;
3126 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3127 }
3128 list_for_each_entry(connector, &mode_config->connector_list, head) {
3129 struct intel_connector *intel_connector = to_intel_connector(connector);
3130 connector->polled = intel_connector->polled;
3131 if (!connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
3132 connector->polled = DRM_CONNECTOR_POLL_HPD;
3133 }
b5ea2d56
DV		 3134
3135 /* Interrupt setup is already guaranteed to be single-threaded, this is
3136 * just to make the assert_spin_locked checks happy. */
3137 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
20afbda2
DV		 3138 if (dev_priv->display.hpd_irq_setup)
3139 dev_priv->display.hpd_irq_setup(dev);
b5ea2d56 3140 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
20afbda2 3141}
Ubuntu Hirsute Linux kernel mirror