]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/gpu/drm/i915/i915_irq.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Do remaps for all contexts
[mirror_ubuntu-hirsute-kernel.git] / drivers / gpu / drm / i915 / i915_irq.c
CommitLineData
0d6aa60b 1/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
1da177e4 2 */
0d6aa60b 3/*
1da177e4
LT		 4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
bc54fd1a
DA		 6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
0d6aa60b 27 */
1da177e4 28
a70491cc
JP		 29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
63eeaf38 31#include <linux/sysrq.h>
5a0e3ad6 32#include <linux/slab.h>
760285e7
DH		 33#include <drm/drmP.h>
34#include <drm/i915_drm.h>
1da177e4 35#include "i915_drv.h"
1c5d22f7 36#include "i915_trace.h"
79e53945 37#include "intel_drv.h"
1da177e4 38
e5868a31
EE		 39static const u32 hpd_ibx[] = {
40 [HPD_CRT] = SDE_CRT_HOTPLUG,
41 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
42 [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
43 [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
44 [HPD_PORT_D] = SDE_PORTD_HOTPLUG
45};
46
47static const u32 hpd_cpt[] = {
48 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
73c352a2 49 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
e5868a31
EE		 50 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
51 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
52 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
53};
54
55static const u32 hpd_mask_i915[] = {
56 [HPD_CRT] = CRT_HOTPLUG_INT_EN,
57 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
58 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
59 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
60 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
61 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
62};
63
64static const u32 hpd_status_gen4[] = {
65 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
66 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
67 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
68 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
69 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
70 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
71};
72
e5868a31
EE		 73static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
74 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
75 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
76 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
77 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
78 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
79 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
80};
81
036a4a7d 82/* For display hotplug interrupt */
995b6762 83static void
f2b115e6 84ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
036a4a7d 85{
4bc9d430
DV		 86 assert_spin_locked(&dev_priv->irq_lock);
87
c67a470b
PZ		 88 if (dev_priv->pc8.irqs_disabled) {
89 WARN(1, "IRQs disabled\n");
90 dev_priv->pc8.regsave.deimr &= ~mask;
91 return;
92 }
93
1ec14ad3
CW		 94 if ((dev_priv->irq_mask & mask) != 0) {
95 dev_priv->irq_mask &= ~mask;
96 I915_WRITE(DEIMR, dev_priv->irq_mask);
3143a2bf 97 POSTING_READ(DEIMR);
036a4a7d
ZW		 98 }
99}
100
0ff9800a 101static void
f2b115e6 102ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
036a4a7d 103{
4bc9d430
DV		 104 assert_spin_locked(&dev_priv->irq_lock);
105
c67a470b
PZ		 106 if (dev_priv->pc8.irqs_disabled) {
107 WARN(1, "IRQs disabled\n");
108 dev_priv->pc8.regsave.deimr |= mask;
109 return;
110 }
111
1ec14ad3
CW		 112 if ((dev_priv->irq_mask & mask) != mask) {
113 dev_priv->irq_mask |= mask;
114 I915_WRITE(DEIMR, dev_priv->irq_mask);
3143a2bf 115 POSTING_READ(DEIMR);
036a4a7d
ZW		 116 }
117}
118
43eaea13
PZ		 119/**
120 * ilk_update_gt_irq - update GTIMR
121 * @dev_priv: driver private
122 * @interrupt_mask: mask of interrupt bits to update
123 * @enabled_irq_mask: mask of interrupt bits to enable
124 */
125static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
126 uint32_t interrupt_mask,
127 uint32_t enabled_irq_mask)
128{
129 assert_spin_locked(&dev_priv->irq_lock);
130
c67a470b
PZ		 131 if (dev_priv->pc8.irqs_disabled) {
132 WARN(1, "IRQs disabled\n");
133 dev_priv->pc8.regsave.gtimr &= ~interrupt_mask;
134 dev_priv->pc8.regsave.gtimr |= (~enabled_irq_mask &
135 interrupt_mask);
136 return;
137 }
138
43eaea13
PZ		 139 dev_priv->gt_irq_mask &= ~interrupt_mask;
140 dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
141 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
142 POSTING_READ(GTIMR);
143}
144
145void ilk_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
146{
147 ilk_update_gt_irq(dev_priv, mask, mask);
148}
149
150void ilk_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
151{
152 ilk_update_gt_irq(dev_priv, mask, 0);
153}
154
edbfdb45
PZ		 155/**
156 * snb_update_pm_irq - update GEN6_PMIMR
157 * @dev_priv: driver private
158 * @interrupt_mask: mask of interrupt bits to update
159 * @enabled_irq_mask: mask of interrupt bits to enable
160 */
161static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
162 uint32_t interrupt_mask,
163 uint32_t enabled_irq_mask)
164{
605cd25b 165 uint32_t new_val;
edbfdb45
PZ		 166
167 assert_spin_locked(&dev_priv->irq_lock);
168
c67a470b
PZ		 169 if (dev_priv->pc8.irqs_disabled) {
170 WARN(1, "IRQs disabled\n");
171 dev_priv->pc8.regsave.gen6_pmimr &= ~interrupt_mask;
172 dev_priv->pc8.regsave.gen6_pmimr |= (~enabled_irq_mask &
173 interrupt_mask);
174 return;
175 }
176
605cd25b 177 new_val = dev_priv->pm_irq_mask;
f52ecbcf
PZ		 178 new_val &= ~interrupt_mask;
179 new_val |= (~enabled_irq_mask & interrupt_mask);
180
605cd25b
PZ		 181 if (new_val != dev_priv->pm_irq_mask) {
182 dev_priv->pm_irq_mask = new_val;
183 I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
f52ecbcf
PZ		 184 POSTING_READ(GEN6_PMIMR);
185 }
edbfdb45
PZ		 186}
187
188void snb_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
189{
190 snb_update_pm_irq(dev_priv, mask, mask);
191}
192
193void snb_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
194{
195 snb_update_pm_irq(dev_priv, mask, 0);
196}
197
8664281b
PZ		 198static bool ivb_can_enable_err_int(struct drm_device *dev)
199{
200 struct drm_i915_private *dev_priv = dev->dev_private;
201 struct intel_crtc *crtc;
202 enum pipe pipe;
203
4bc9d430
DV		 204 assert_spin_locked(&dev_priv->irq_lock);
205
8664281b
PZ		 206 for_each_pipe(pipe) {
207 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
208
209 if (crtc->cpu_fifo_underrun_disabled)
210 return false;
211 }
212
213 return true;
214}
215
216static bool cpt_can_enable_serr_int(struct drm_device *dev)
217{
218 struct drm_i915_private *dev_priv = dev->dev_private;
219 enum pipe pipe;
220 struct intel_crtc *crtc;
221
fee884ed
DV		 222 assert_spin_locked(&dev_priv->irq_lock);
223
8664281b
PZ		 224 for_each_pipe(pipe) {
225 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
226
227 if (crtc->pch_fifo_underrun_disabled)
228 return false;
229 }
230
231 return true;
232}
233
234static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
235 enum pipe pipe, bool enable)
236{
237 struct drm_i915_private *dev_priv = dev->dev_private;
238 uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
239 DE_PIPEB_FIFO_UNDERRUN;
240
241 if (enable)
242 ironlake_enable_display_irq(dev_priv, bit);
243 else
244 ironlake_disable_display_irq(dev_priv, bit);
245}
246
247static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
7336df65 248 enum pipe pipe, bool enable)
8664281b
PZ		 249{
250 struct drm_i915_private *dev_priv = dev->dev_private;
8664281b 251 if (enable) {
7336df65
DV		 252 I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe));
253
8664281b
PZ		 254 if (!ivb_can_enable_err_int(dev))
255 return;
256
8664281b
PZ		 257 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
258 } else {
7336df65
DV		 259 bool was_enabled = !(I915_READ(DEIMR) & DE_ERR_INT_IVB);
260
261 /* Change the state _after_ we've read out the current one. */
8664281b 262 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
7336df65
DV		 263
264 if (!was_enabled &&
265 (I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe))) {
266 DRM_DEBUG_KMS("uncleared fifo underrun on pipe %c\n",
267 pipe_name(pipe));
268 }
8664281b
PZ		 269 }
270}
271
fee884ed
DV		 272/**
273 * ibx_display_interrupt_update - update SDEIMR
274 * @dev_priv: driver private
275 * @interrupt_mask: mask of interrupt bits to update
276 * @enabled_irq_mask: mask of interrupt bits to enable
277 */
278static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
279 uint32_t interrupt_mask,
280 uint32_t enabled_irq_mask)
281{
282 uint32_t sdeimr = I915_READ(SDEIMR);
283 sdeimr &= ~interrupt_mask;
284 sdeimr |= (~enabled_irq_mask & interrupt_mask);
285
286 assert_spin_locked(&dev_priv->irq_lock);
287
c67a470b
PZ		 288 if (dev_priv->pc8.irqs_disabled &&
289 (interrupt_mask & SDE_HOTPLUG_MASK_CPT)) {
290 WARN(1, "IRQs disabled\n");
291 dev_priv->pc8.regsave.sdeimr &= ~interrupt_mask;
292 dev_priv->pc8.regsave.sdeimr |= (~enabled_irq_mask &
293 interrupt_mask);
294 return;
295 }
296
fee884ed
DV		 297 I915_WRITE(SDEIMR, sdeimr);
298 POSTING_READ(SDEIMR);
299}
300#define ibx_enable_display_interrupt(dev_priv, bits) \
301 ibx_display_interrupt_update((dev_priv), (bits), (bits))
302#define ibx_disable_display_interrupt(dev_priv, bits) \
303 ibx_display_interrupt_update((dev_priv), (bits), 0)
304
de28075d
DV		 305static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
306 enum transcoder pch_transcoder,
8664281b
PZ		 307 bool enable)
308{
8664281b 309 struct drm_i915_private *dev_priv = dev->dev_private;
de28075d
DV		 310 uint32_t bit = (pch_transcoder == TRANSCODER_A) ?
311 SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER;
8664281b
PZ		 312
313 if (enable)
fee884ed 314 ibx_enable_display_interrupt(dev_priv, bit);
8664281b 315 else
fee884ed 316 ibx_disable_display_interrupt(dev_priv, bit);
8664281b
PZ		 317}
318
319static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
320 enum transcoder pch_transcoder,
321 bool enable)
322{
323 struct drm_i915_private *dev_priv = dev->dev_private;
324
325 if (enable) {
1dd246fb
DV		 326 I915_WRITE(SERR_INT,
327 SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder));
328
8664281b
PZ		 329 if (!cpt_can_enable_serr_int(dev))
330 return;
331
fee884ed 332 ibx_enable_display_interrupt(dev_priv, SDE_ERROR_CPT);
8664281b 333 } else {
1dd246fb
DV		 334 uint32_t tmp = I915_READ(SERR_INT);
335 bool was_enabled = !(I915_READ(SDEIMR) & SDE_ERROR_CPT);
336
337 /* Change the state _after_ we've read out the current one. */
fee884ed 338 ibx_disable_display_interrupt(dev_priv, SDE_ERROR_CPT);
1dd246fb
DV		 339
340 if (!was_enabled &&
341 (tmp & SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder))) {
342 DRM_DEBUG_KMS("uncleared pch fifo underrun on pch transcoder %c\n",
343 transcoder_name(pch_transcoder));
344 }
8664281b 345 }
8664281b
PZ		 346}
347
348/**
349 * intel_set_cpu_fifo_underrun_reporting - enable/disable FIFO underrun messages
350 * @dev: drm device
351 * @pipe: pipe
352 * @enable: true if we want to report FIFO underrun errors, false otherwise
353 *
354 * This function makes us disable or enable CPU fifo underruns for a specific
355 * pipe. Notice that on some Gens (e.g. IVB, HSW), disabling FIFO underrun
356 * reporting for one pipe may also disable all the other CPU error interruts for
357 * the other pipes, due to the fact that there's just one interrupt mask/enable
358 * bit for all the pipes.
359 *
360 * Returns the previous state of underrun reporting.
361 */
362bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
363 enum pipe pipe, bool enable)
364{
365 struct drm_i915_private *dev_priv = dev->dev_private;
366 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
367 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
368 unsigned long flags;
369 bool ret;
370
371 spin_lock_irqsave(&dev_priv->irq_lock, flags);
372
373 ret = !intel_crtc->cpu_fifo_underrun_disabled;
374
375 if (enable == ret)
376 goto done;
377
378 intel_crtc->cpu_fifo_underrun_disabled = !enable;
379
380 if (IS_GEN5(dev) || IS_GEN6(dev))
381 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
382 else if (IS_GEN7(dev))
7336df65 383 ivybridge_set_fifo_underrun_reporting(dev, pipe, enable);
8664281b
PZ		 384
385done:
386 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
387 return ret;
388}
389
390/**
391 * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
392 * @dev: drm device
393 * @pch_transcoder: the PCH transcoder (same as pipe on IVB and older)
394 * @enable: true if we want to report FIFO underrun errors, false otherwise
395 *
396 * This function makes us disable or enable PCH fifo underruns for a specific
397 * PCH transcoder. Notice that on some PCHs (e.g. CPT/PPT), disabling FIFO
398 * underrun reporting for one transcoder may also disable all the other PCH
399 * error interruts for the other transcoders, due to the fact that there's just
400 * one interrupt mask/enable bit for all the transcoders.
401 *
402 * Returns the previous state of underrun reporting.
403 */
404bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
405 enum transcoder pch_transcoder,
406 bool enable)
407{
408 struct drm_i915_private *dev_priv = dev->dev_private;
de28075d
DV		 409 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
410 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8664281b
PZ		 411 unsigned long flags;
412 bool ret;
413
de28075d
DV		 414 /*
415 * NOTE: Pre-LPT has a fixed cpu pipe -> pch transcoder mapping, but LPT
416 * has only one pch transcoder A that all pipes can use. To avoid racy
417 * pch transcoder -> pipe lookups from interrupt code simply store the
418 * underrun statistics in crtc A. Since we never expose this anywhere
419 * nor use it outside of the fifo underrun code here using the "wrong"
420 * crtc on LPT won't cause issues.
421 */
8664281b
PZ		 422
423 spin_lock_irqsave(&dev_priv->irq_lock, flags);
424
425 ret = !intel_crtc->pch_fifo_underrun_disabled;
426
427 if (enable == ret)
428 goto done;
429
430 intel_crtc->pch_fifo_underrun_disabled = !enable;
431
432 if (HAS_PCH_IBX(dev))
de28075d 433 ibx_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
8664281b
PZ		 434 else
435 cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
436
437done:
438 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
439 return ret;
440}
441
442
7c463586
KP		 443void
444i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
445{
46c06a30
VS		 446 u32 reg = PIPESTAT(pipe);
447 u32 pipestat = I915_READ(reg) & 0x7fff0000;
7c463586 448
b79480ba
DV		 449 assert_spin_locked(&dev_priv->irq_lock);
450
46c06a30
VS		 451 if ((pipestat & mask) == mask)
452 return;
453
454 /* Enable the interrupt, clear any pending status */
455 pipestat |= mask | (mask >> 16);
456 I915_WRITE(reg, pipestat);
457 POSTING_READ(reg);
7c463586
KP		 458}
459
460void
461i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
462{
46c06a30
VS		 463 u32 reg = PIPESTAT(pipe);
464 u32 pipestat = I915_READ(reg) & 0x7fff0000;
7c463586 465
b79480ba
DV		 466 assert_spin_locked(&dev_priv->irq_lock);
467
46c06a30
VS		 468 if ((pipestat & mask) == 0)
469 return;
470
471 pipestat &= ~mask;
472 I915_WRITE(reg, pipestat);
473 POSTING_READ(reg);
7c463586
KP		 474}
475
01c66889 476/**
f49e38dd 477 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
01c66889 478 */
f49e38dd 479static void i915_enable_asle_pipestat(struct drm_device *dev)
01c66889 480{
1ec14ad3
CW		 481 drm_i915_private_t *dev_priv = dev->dev_private;
482 unsigned long irqflags;
483
f49e38dd
JN		 484 if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
485 return;
486
1ec14ad3 487 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
01c66889 488
f898780b
JN		 489 i915_enable_pipestat(dev_priv, 1, PIPE_LEGACY_BLC_EVENT_ENABLE);
490 if (INTEL_INFO(dev)->gen >= 4)
491 i915_enable_pipestat(dev_priv, 0, PIPE_LEGACY_BLC_EVENT_ENABLE);
1ec14ad3
CW		 492
493 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
01c66889
ZY		 494}
495
0a3e67a4
JB		 496/**
497 * i915_pipe_enabled - check if a pipe is enabled
498 * @dev: DRM device
499 * @pipe: pipe to check
500 *
501 * Reading certain registers when the pipe is disabled can hang the chip.
502 * Use this routine to make sure the PLL is running and the pipe is active
503 * before reading such registers if unsure.
504 */
505static int
506i915_pipe_enabled(struct drm_device *dev, int pipe)
507{
508 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
702e7a56 509
a01025af
DV		 510 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
511 /* Locking is horribly broken here, but whatever. */
512 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
513 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
71f8ba6b 514
a01025af
DV		 515 return intel_crtc->active;
516 } else {
517 return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
518 }
0a3e67a4
JB		 519}
520
42f52ef8
KP		 521/* Called from drm generic code, passed a 'crtc', which
522 * we use as a pipe index
523 */
f71d4af4 524static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
0a3e67a4
JB		 525{
526 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
527 unsigned long high_frame;
528 unsigned long low_frame;
5eddb70b 529 u32 high1, high2, low;
0a3e67a4
JB		 530
531 if (!i915_pipe_enabled(dev, pipe)) {
44d98a61 532 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
9db4a9c7 533 "pipe %c\n", pipe_name(pipe));
0a3e67a4
JB		 534 return 0;
535 }
536
9db4a9c7
JB		 537 high_frame = PIPEFRAME(pipe);
538 low_frame = PIPEFRAMEPIXEL(pipe);
5eddb70b 539
0a3e67a4
JB		 540 /*
541 * High & low register fields aren't synchronized, so make sure
542 * we get a low value that's stable across two reads of the high
543 * register.
544 */
545 do {
5eddb70b
CW		 546 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
547 low = I915_READ(low_frame) & PIPE_FRAME_LOW_MASK;
548 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
0a3e67a4
JB		 549 } while (high1 != high2);
550
5eddb70b
CW		 551 high1 >>= PIPE_FRAME_HIGH_SHIFT;
552 low >>= PIPE_FRAME_LOW_SHIFT;
553 return (high1 << 8) | low;
0a3e67a4
JB		 554}
555
f71d4af4 556static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
9880b7a5
JB		 557{
558 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
9db4a9c7 559 int reg = PIPE_FRMCOUNT_GM45(pipe);
9880b7a5
JB		 560
561 if (!i915_pipe_enabled(dev, pipe)) {
44d98a61 562 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
9db4a9c7 563 "pipe %c\n", pipe_name(pipe));
9880b7a5
JB		 564 return 0;
565 }
566
567 return I915_READ(reg);
568}
569
f71d4af4 570static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
0af7e4df
MK		 571 int *vpos, int *hpos)
572{
573 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
574 u32 vbl = 0, position = 0;
575 int vbl_start, vbl_end, htotal, vtotal;
576 bool in_vbl = true;
577 int ret = 0;
fe2b8f9d
PZ		 578 enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
579 pipe);
0af7e4df
MK		 580
581 if (!i915_pipe_enabled(dev, pipe)) {
582 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
9db4a9c7 583 "pipe %c\n", pipe_name(pipe));
0af7e4df
MK		 584 return 0;
585 }
586
587 /* Get vtotal. */
fe2b8f9d 588 vtotal = 1 + ((I915_READ(VTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
0af7e4df
MK		 589
590 if (INTEL_INFO(dev)->gen >= 4) {
591 /* No obvious pixelcount register. Only query vertical
592 * scanout position from Display scan line register.
593 */
594 position = I915_READ(PIPEDSL(pipe));
595
596 /* Decode into vertical scanout position. Don't have
597 * horizontal scanout position.
598 */
599 *vpos = position & 0x1fff;
600 *hpos = 0;
601 } else {
602 /* Have access to pixelcount since start of frame.
603 * We can split this into vertical and horizontal
604 * scanout position.
605 */
606 position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
607
fe2b8f9d 608 htotal = 1 + ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
0af7e4df
MK		 609 *vpos = position / htotal;
610 *hpos = position - (*vpos * htotal);
611 }
612
613 /* Query vblank area. */
fe2b8f9d 614 vbl = I915_READ(VBLANK(cpu_transcoder));
0af7e4df
MK		 615
616 /* Test position against vblank region. */
617 vbl_start = vbl & 0x1fff;
618 vbl_end = (vbl >> 16) & 0x1fff;
619
620 if ((*vpos < vbl_start) || (*vpos > vbl_end))
621 in_vbl = false;
622
623 /* Inside "upper part" of vblank area? Apply corrective offset: */
624 if (in_vbl && (*vpos >= vbl_start))
625 *vpos = *vpos - vtotal;
626
627 /* Readouts valid? */
628 if (vbl > 0)
629 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
630
631 /* In vblank? */
632 if (in_vbl)
633 ret |= DRM_SCANOUTPOS_INVBL;
634
635 return ret;
636}
637
f71d4af4 638static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
0af7e4df
MK		 639 int *max_error,
640 struct timeval *vblank_time,
641 unsigned flags)
642{
4041b853 643 struct drm_crtc *crtc;
0af7e4df 644
7eb552ae 645 if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
4041b853 646 DRM_ERROR("Invalid crtc %d\n", pipe);
0af7e4df
MK		 647 return -EINVAL;
648 }
649
650 /* Get drm_crtc to timestamp: */
4041b853
CW		 651 crtc = intel_get_crtc_for_pipe(dev, pipe);
652 if (crtc == NULL) {
653 DRM_ERROR("Invalid crtc %d\n", pipe);
654 return -EINVAL;
655 }
656
657 if (!crtc->enabled) {
658 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
659 return -EBUSY;
660 }
0af7e4df
MK		 661
662 /* Helper routine in DRM core does all the work: */
4041b853
CW		 663 return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
664 vblank_time, flags,
665 crtc);
0af7e4df
MK		 666}
667
67c347ff
JN		 668static bool intel_hpd_irq_event(struct drm_device *dev,
669 struct drm_connector *connector)
321a1b30
EE		 670{
671 enum drm_connector_status old_status;
672
673 WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
674 old_status = connector->status;
675
676 connector->status = connector->funcs->detect(connector, false);
67c347ff
JN		 677 if (old_status == connector->status)
678 return false;
679
680 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
321a1b30
EE		 681 connector->base.id,
682 drm_get_connector_name(connector),
67c347ff
JN		 683 drm_get_connector_status_name(old_status),
684 drm_get_connector_status_name(connector->status));
685
686 return true;
321a1b30
EE		 687}
688
5ca58282
JB		 689/*
690 * Handle hotplug events outside the interrupt handler proper.
691 */
ac4c16c5
EE		 692#define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)
693
5ca58282
JB		 694static void i915_hotplug_work_func(struct work_struct *work)
695{
696 drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
697 hotplug_work);
698 struct drm_device *dev = dev_priv->dev;
c31c4ba3 699 struct drm_mode_config *mode_config = &dev->mode_config;
cd569aed
EE		 700 struct intel_connector *intel_connector;
701 struct intel_encoder *intel_encoder;
702 struct drm_connector *connector;
703 unsigned long irqflags;
704 bool hpd_disabled = false;
321a1b30 705 bool changed = false;
142e2398 706 u32 hpd_event_bits;
4ef69c7a 707
52d7eced
DV		 708 /* HPD irq before everything is fully set up. */
709 if (!dev_priv->enable_hotplug_processing)
710 return;
711
a65e34c7 712 mutex_lock(&mode_config->mutex);
e67189ab
JB		 713 DRM_DEBUG_KMS("running encoder hotplug functions\n");
714
cd569aed 715 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
142e2398
EE		 716
717 hpd_event_bits = dev_priv->hpd_event_bits;
718 dev_priv->hpd_event_bits = 0;
cd569aed
EE		 719 list_for_each_entry(connector, &mode_config->connector_list, head) {
720 intel_connector = to_intel_connector(connector);
721 intel_encoder = intel_connector->encoder;
722 if (intel_encoder->hpd_pin > HPD_NONE &&
723 dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
724 connector->polled == DRM_CONNECTOR_POLL_HPD) {
725 DRM_INFO("HPD interrupt storm detected on connector %s: "
726 "switching from hotplug detection to polling\n",
727 drm_get_connector_name(connector));
728 dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
729 connector->polled = DRM_CONNECTOR_POLL_CONNECT
730 | DRM_CONNECTOR_POLL_DISCONNECT;
731 hpd_disabled = true;
732 }
142e2398
EE		 733 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
734 DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
735 drm_get_connector_name(connector), intel_encoder->hpd_pin);
736 }
cd569aed
EE		 737 }
738 /* if there were no outputs to poll, poll was disabled,
739 * therefore make sure it's enabled when disabling HPD on
740 * some connectors */
ac4c16c5 741 if (hpd_disabled) {
cd569aed 742 drm_kms_helper_poll_enable(dev);
ac4c16c5
EE		 743 mod_timer(&dev_priv->hotplug_reenable_timer,
744 jiffies + msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
745 }
cd569aed
EE		 746
747 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
748
321a1b30
EE		 749 list_for_each_entry(connector, &mode_config->connector_list, head) {
750 intel_connector = to_intel_connector(connector);
751 intel_encoder = intel_connector->encoder;
752 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
753 if (intel_encoder->hot_plug)
754 intel_encoder->hot_plug(intel_encoder);
755 if (intel_hpd_irq_event(dev, connector))
756 changed = true;
757 }
758 }
40ee3381
KP		 759 mutex_unlock(&mode_config->mutex);
760
321a1b30
EE		 761 if (changed)
762 drm_kms_helper_hotplug_event(dev);
5ca58282
JB		 763}
764
d0ecd7e2 765static void ironlake_rps_change_irq_handler(struct drm_device *dev)
f97108d1
JB		 766{
767 drm_i915_private_t *dev_priv = dev->dev_private;
b5b72e89 768 u32 busy_up, busy_down, max_avg, min_avg;
9270388e 769 u8 new_delay;
9270388e 770
d0ecd7e2 771 spin_lock(&mchdev_lock);
f97108d1 772
73edd18f
DV		 773 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
774
20e4d407 775 new_delay = dev_priv->ips.cur_delay;
9270388e 776
7648fa99 777 I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
b5b72e89
MG		 778 busy_up = I915_READ(RCPREVBSYTUPAVG);
779 busy_down = I915_READ(RCPREVBSYTDNAVG);
f97108d1
JB		 780 max_avg = I915_READ(RCBMAXAVG);
781 min_avg = I915_READ(RCBMINAVG);
782
783 /* Handle RCS change request from hw */
b5b72e89 784 if (busy_up > max_avg) {
20e4d407
DV		 785 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
786 new_delay = dev_priv->ips.cur_delay - 1;
787 if (new_delay < dev_priv->ips.max_delay)
788 new_delay = dev_priv->ips.max_delay;
b5b72e89 789 } else if (busy_down < min_avg) {
20e4d407
DV		 790 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
791 new_delay = dev_priv->ips.cur_delay + 1;
792 if (new_delay > dev_priv->ips.min_delay)
793 new_delay = dev_priv->ips.min_delay;
f97108d1
JB		 794 }
795
7648fa99 796 if (ironlake_set_drps(dev, new_delay))
20e4d407 797 dev_priv->ips.cur_delay = new_delay;
f97108d1 798
d0ecd7e2 799 spin_unlock(&mchdev_lock);
9270388e 800
f97108d1
JB		 801 return;
802}
803
549f7365
CW		 804static void notify_ring(struct drm_device *dev,
805 struct intel_ring_buffer *ring)
806{
475553de
CW		 807 if (ring->obj == NULL)
808 return;
809
b2eadbc8 810 trace_i915_gem_request_complete(ring, ring->get_seqno(ring, false));
9862e600 811
549f7365 812 wake_up_all(&ring->irq_queue);
10cd45b6 813 i915_queue_hangcheck(dev);
549f7365
CW		 814}
815
4912d041 816static void gen6_pm_rps_work(struct work_struct *work)
3b8d8d91 817{
4912d041 818 drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
c6a828d3 819 rps.work);
edbfdb45 820 u32 pm_iir;
7b9e0ae6 821 u8 new_delay;
4912d041 822
59cdb63d 823 spin_lock_irq(&dev_priv->irq_lock);
c6a828d3
DV		 824 pm_iir = dev_priv->rps.pm_iir;
825 dev_priv->rps.pm_iir = 0;
4848405c 826 /* Make sure not to corrupt PMIMR state used by ringbuffer code */
edbfdb45 827 snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
59cdb63d 828 spin_unlock_irq(&dev_priv->irq_lock);
3b8d8d91 829
60611c13
PZ		 830 /* Make sure we didn't queue anything we're not going to process. */
831 WARN_ON(pm_iir & ~GEN6_PM_RPS_EVENTS);
832
4848405c 833 if ((pm_iir & GEN6_PM_RPS_EVENTS) == 0)
3b8d8d91
JB		 834 return;
835
4fc688ce 836 mutex_lock(&dev_priv->rps.hw_lock);
7b9e0ae6 837
7425034a 838 if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
c6a828d3 839 new_delay = dev_priv->rps.cur_delay + 1;
7425034a
VS		 840
841 /*
842 * For better performance, jump directly
843 * to RPe if we're below it.
844 */
845 if (IS_VALLEYVIEW(dev_priv->dev) &&
846 dev_priv->rps.cur_delay < dev_priv->rps.rpe_delay)
847 new_delay = dev_priv->rps.rpe_delay;
848 } else
c6a828d3 849 new_delay = dev_priv->rps.cur_delay - 1;
3b8d8d91 850
79249636
BW		 851 /* sysfs frequency interfaces may have snuck in while servicing the
852 * interrupt
853 */
d8289c9e
VS		 854 if (new_delay >= dev_priv->rps.min_delay &&
855 new_delay <= dev_priv->rps.max_delay) {
0a073b84
JB		 856 if (IS_VALLEYVIEW(dev_priv->dev))
857 valleyview_set_rps(dev_priv->dev, new_delay);
858 else
859 gen6_set_rps(dev_priv->dev, new_delay);
79249636 860 }
3b8d8d91 861
52ceb908
JB		 862 if (IS_VALLEYVIEW(dev_priv->dev)) {
863 /*
864 * On VLV, when we enter RC6 we may not be at the minimum
865 * voltage level, so arm a timer to check. It should only
866 * fire when there's activity or once after we've entered
867 * RC6, and then won't be re-armed until the next RPS interrupt.
868 */
869 mod_delayed_work(dev_priv->wq, &dev_priv->rps.vlv_work,
870 msecs_to_jiffies(100));
871 }
872
4fc688ce 873 mutex_unlock(&dev_priv->rps.hw_lock);
3b8d8d91
JB		 874}
875
e3689190
BW		 876
877/**
878 * ivybridge_parity_work - Workqueue called when a parity error interrupt
879 * occurred.
880 * @work: workqueue struct
881 *
882 * Doesn't actually do anything except notify userspace. As a consequence of
883 * this event, userspace should try to remap the bad rows since statistically
884 * it is likely the same row is more likely to go bad again.
885 */
886static void ivybridge_parity_work(struct work_struct *work)
887{
888 drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
a4da4fa4 889 l3_parity.error_work);
e3689190 890 u32 error_status, row, bank, subbank;
35a85ac6 891 char *parity_event[6];
e3689190
BW		 892 uint32_t misccpctl;
893 unsigned long flags;
35a85ac6 894 uint8_t slice = 0;
e3689190
BW		 895
896 /* We must turn off DOP level clock gating to access the L3 registers.
897 * In order to prevent a get/put style interface, acquire struct mutex
898 * any time we access those registers.
899 */
900 mutex_lock(&dev_priv->dev->struct_mutex);
901
35a85ac6
BW		 902 /* If we've screwed up tracking, just let the interrupt fire again */
903 if (WARN_ON(!dev_priv->l3_parity.which_slice))
904 goto out;
905
e3689190
BW		 906 misccpctl = I915_READ(GEN7_MISCCPCTL);
907 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
908 POSTING_READ(GEN7_MISCCPCTL);
909
35a85ac6
BW		 910 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
911 u32 reg;
e3689190 912
35a85ac6
BW		 913 slice--;
914 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
915 break;
e3689190 916
35a85ac6 917 dev_priv->l3_parity.which_slice &= ~(1<<slice);
e3689190 918
35a85ac6 919 reg = GEN7_L3CDERRST1 + (slice * 0x200);
e3689190 920
35a85ac6
BW		 921 error_status = I915_READ(reg);
922 row = GEN7_PARITY_ERROR_ROW(error_status);
923 bank = GEN7_PARITY_ERROR_BANK(error_status);
924 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
925
926 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
927 POSTING_READ(reg);
928
929 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
930 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
931 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
932 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
933 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
934 parity_event[5] = NULL;
935
936 kobject_uevent_env(&dev_priv->dev->primary->kdev.kobj,
937 KOBJ_CHANGE, parity_event);
e3689190 938
35a85ac6
BW		 939 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
940 slice, row, bank, subbank);
e3689190 941
35a85ac6
BW		 942 kfree(parity_event[4]);
943 kfree(parity_event[3]);
944 kfree(parity_event[2]);
945 kfree(parity_event[1]);
946 }
e3689190 947
35a85ac6 948 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
e3689190 949
35a85ac6
BW		 950out:
951 WARN_ON(dev_priv->l3_parity.which_slice);
952 spin_lock_irqsave(&dev_priv->irq_lock, flags);
953 ilk_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
954 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
955
956 mutex_unlock(&dev_priv->dev->struct_mutex);
e3689190
BW		 957}
958
35a85ac6 959static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
e3689190
BW		 960{
961 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
e3689190 962
e1ef7cc2 963 if (!HAS_L3_GPU_CACHE(dev))
e3689190
BW		 964 return;
965
d0ecd7e2 966 spin_lock(&dev_priv->irq_lock);
35a85ac6 967 ilk_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
d0ecd7e2 968 spin_unlock(&dev_priv->irq_lock);
e3689190 969
35a85ac6
BW		 970 iir &= GT_PARITY_ERROR(dev);
971 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
972 dev_priv->l3_parity.which_slice |= 1 << 1;
973
974 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
975 dev_priv->l3_parity.which_slice |= 1 << 0;
976
a4da4fa4 977 queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
e3689190
BW		 978}
979
f1af8fc1
PZ		 980static void ilk_gt_irq_handler(struct drm_device *dev,
981 struct drm_i915_private *dev_priv,
982 u32 gt_iir)
983{
984 if (gt_iir &
985 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
986 notify_ring(dev, &dev_priv->ring[RCS]);
987 if (gt_iir & ILK_BSD_USER_INTERRUPT)
988 notify_ring(dev, &dev_priv->ring[VCS]);
989}
990
e7b4c6b1
DV		 991static void snb_gt_irq_handler(struct drm_device *dev,
992 struct drm_i915_private *dev_priv,
993 u32 gt_iir)
994{
995
cc609d5d
BW		 996 if (gt_iir &
997 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
e7b4c6b1 998 notify_ring(dev, &dev_priv->ring[RCS]);
cc609d5d 999 if (gt_iir & GT_BSD_USER_INTERRUPT)
e7b4c6b1 1000 notify_ring(dev, &dev_priv->ring[VCS]);
cc609d5d 1001 if (gt_iir & GT_BLT_USER_INTERRUPT)
e7b4c6b1
DV		 1002 notify_ring(dev, &dev_priv->ring[BCS]);
1003
cc609d5d
BW		 1004 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1005 GT_BSD_CS_ERROR_INTERRUPT |
1006 GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
e7b4c6b1
DV		 1007 DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir);
1008 i915_handle_error(dev, false);
1009 }
e3689190 1010
35a85ac6
BW		 1011 if (gt_iir & GT_PARITY_ERROR(dev))
1012 ivybridge_parity_error_irq_handler(dev, gt_iir);
e7b4c6b1
DV		 1013}
1014
b543fb04
EE		 1015#define HPD_STORM_DETECT_PERIOD 1000
1016#define HPD_STORM_THRESHOLD 5
1017
10a504de 1018static inline void intel_hpd_irq_handler(struct drm_device *dev,
22062dba
DV		 1019 u32 hotplug_trigger,
1020 const u32 *hpd)
b543fb04
EE		 1021{
1022 drm_i915_private_t *dev_priv = dev->dev_private;
b543fb04 1023 int i;
10a504de 1024 bool storm_detected = false;
b543fb04 1025
91d131d2
DV		 1026 if (!hotplug_trigger)
1027 return;
1028
b5ea2d56 1029 spin_lock(&dev_priv->irq_lock);
b543fb04 1030 for (i = 1; i < HPD_NUM_PINS; i++) {
821450c6 1031
b8f102e8
EE		 1032 WARN(((hpd[i] & hotplug_trigger) &&
1033 dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED),
1034 "Received HPD interrupt although disabled\n");
1035
b543fb04
EE		 1036 if (!(hpd[i] & hotplug_trigger) ||
1037 dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
1038 continue;
1039
bc5ead8c 1040 dev_priv->hpd_event_bits |= (1 << i);
b543fb04
EE		 1041 if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
1042 dev_priv->hpd_stats[i].hpd_last_jiffies
1043 + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
1044 dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
1045 dev_priv->hpd_stats[i].hpd_cnt = 0;
b8f102e8 1046 DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i);
b543fb04
EE		 1047 } else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
1048 dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
142e2398 1049 dev_priv->hpd_event_bits &= ~(1 << i);
b543fb04 1050 DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
10a504de 1051 storm_detected = true;
b543fb04
EE		 1052 } else {
1053 dev_priv->hpd_stats[i].hpd_cnt++;
b8f102e8
EE		 1054 DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i,
1055 dev_priv->hpd_stats[i].hpd_cnt);
b543fb04
EE		 1056 }
1057 }
1058
10a504de
DV		 1059 if (storm_detected)
1060 dev_priv->display.hpd_irq_setup(dev);
b5ea2d56 1061 spin_unlock(&dev_priv->irq_lock);
5876fa0d 1062
645416f5
DV		 1063 /*
1064 * Our hotplug handler can grab modeset locks (by calling down into the
1065 * fb helpers). Hence it must not be run on our own dev-priv->wq work
1066 * queue for otherwise the flush_work in the pageflip code will
1067 * deadlock.
1068 */
1069 schedule_work(&dev_priv->hotplug_work);
b543fb04
EE		 1070}
1071
515ac2bb
DV		 1072static void gmbus_irq_handler(struct drm_device *dev)
1073{
28c70f16
DV		 1074 struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
1075
28c70f16 1076 wake_up_all(&dev_priv->gmbus_wait_queue);
515ac2bb
DV		 1077}
1078
ce99c256
DV		 1079static void dp_aux_irq_handler(struct drm_device *dev)
1080{
9ee32fea
DV		 1081 struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
1082
9ee32fea 1083 wake_up_all(&dev_priv->gmbus_wait_queue);
ce99c256
DV		 1084}
1085
1403c0d4
PZ		 1086/* The RPS events need forcewake, so we add them to a work queue and mask their
1087 * IMR bits until the work is done. Other interrupts can be processed without
1088 * the work queue. */
1089static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
baf02a1f 1090{
41a05a3a 1091 if (pm_iir & GEN6_PM_RPS_EVENTS) {
59cdb63d 1092 spin_lock(&dev_priv->irq_lock);
41a05a3a 1093 dev_priv->rps.pm_iir |= pm_iir & GEN6_PM_RPS_EVENTS;
4d3b3d5f 1094 snb_disable_pm_irq(dev_priv, pm_iir & GEN6_PM_RPS_EVENTS);
59cdb63d 1095 spin_unlock(&dev_priv->irq_lock);
2adbee62
DV		 1096
1097 queue_work(dev_priv->wq, &dev_priv->rps.work);
baf02a1f 1098 }
baf02a1f 1099
1403c0d4
PZ		 1100 if (HAS_VEBOX(dev_priv->dev)) {
1101 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1102 notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
12638c57 1103
1403c0d4
PZ		 1104 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
1105 DRM_ERROR("VEBOX CS error interrupt 0x%08x\n", pm_iir);
1106 i915_handle_error(dev_priv->dev, false);
1107 }
12638c57 1108 }
baf02a1f
BW		 1109}
1110
ff1f525e 1111static irqreturn_t valleyview_irq_handler(int irq, void *arg)
7e231dbe
JB		 1112{
1113 struct drm_device *dev = (struct drm_device *) arg;
1114 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1115 u32 iir, gt_iir, pm_iir;
1116 irqreturn_t ret = IRQ_NONE;
1117 unsigned long irqflags;
1118 int pipe;
1119 u32 pipe_stats[I915_MAX_PIPES];
7e231dbe
JB		 1120
1121 atomic_inc(&dev_priv->irq_received);
1122
7e231dbe
JB		 1123 while (true) {
1124 iir = I915_READ(VLV_IIR);
1125 gt_iir = I915_READ(GTIIR);
1126 pm_iir = I915_READ(GEN6_PMIIR);
1127
1128 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1129 goto out;
1130
1131 ret = IRQ_HANDLED;
1132
e7b4c6b1 1133 snb_gt_irq_handler(dev, dev_priv, gt_iir);
7e231dbe
JB		 1134
1135 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1136 for_each_pipe(pipe) {
1137 int reg = PIPESTAT(pipe);
1138 pipe_stats[pipe] = I915_READ(reg);
1139
1140 /*
1141 * Clear the PIPE*STAT regs before the IIR
1142 */
1143 if (pipe_stats[pipe] & 0x8000ffff) {
1144 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1145 DRM_DEBUG_DRIVER("pipe %c underrun\n",
1146 pipe_name(pipe));
1147 I915_WRITE(reg, pipe_stats[pipe]);
1148 }
1149 }
1150 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1151
31acc7f5
JB		 1152 for_each_pipe(pipe) {
1153 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1154 drm_handle_vblank(dev, pipe);
1155
1156 if (pipe_stats[pipe] & PLANE_FLIPDONE_INT_STATUS_VLV) {
1157 intel_prepare_page_flip(dev, pipe);
1158 intel_finish_page_flip(dev, pipe);
1159 }
1160 }
1161
7e231dbe
JB		 1162 /* Consume port. Then clear IIR or we'll miss events */
1163 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
1164 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
b543fb04 1165 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
7e231dbe
JB		 1166
1167 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
1168 hotplug_status);
91d131d2
DV		 1169
1170 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
1171
7e231dbe
JB		 1172 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1173 I915_READ(PORT_HOTPLUG_STAT);
1174 }
1175
515ac2bb
DV		 1176 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1177 gmbus_irq_handler(dev);
7e231dbe 1178
60611c13 1179 if (pm_iir)
d0ecd7e2 1180 gen6_rps_irq_handler(dev_priv, pm_iir);
7e231dbe
JB		 1181
1182 I915_WRITE(GTIIR, gt_iir);
1183 I915_WRITE(GEN6_PMIIR, pm_iir);
1184 I915_WRITE(VLV_IIR, iir);
1185 }
1186
1187out:
1188 return ret;
1189}
1190
23e81d69 1191static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
776ad806
JB		 1192{
1193 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
9db4a9c7 1194 int pipe;
b543fb04 1195 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
776ad806 1196
91d131d2
DV		 1197 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1198
cfc33bf7
VS		 1199 if (pch_iir & SDE_AUDIO_POWER_MASK) {
1200 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1201 SDE_AUDIO_POWER_SHIFT);
776ad806 1202 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
cfc33bf7
VS		 1203 port_name(port));
1204 }
776ad806 1205
ce99c256
DV		 1206 if (pch_iir & SDE_AUX_MASK)
1207 dp_aux_irq_handler(dev);
1208
776ad806 1209 if (pch_iir & SDE_GMBUS)
515ac2bb 1210 gmbus_irq_handler(dev);
776ad806
JB		 1211
1212 if (pch_iir & SDE_AUDIO_HDCP_MASK)
1213 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1214
1215 if (pch_iir & SDE_AUDIO_TRANS_MASK)
1216 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1217
1218 if (pch_iir & SDE_POISON)
1219 DRM_ERROR("PCH poison interrupt\n");
1220
9db4a9c7
JB		 1221 if (pch_iir & SDE_FDI_MASK)
1222 for_each_pipe(pipe)
1223 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1224 pipe_name(pipe),
1225 I915_READ(FDI_RX_IIR(pipe)));
776ad806
JB		 1226
1227 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1228 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1229
1230 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1231 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1232
776ad806 1233 if (pch_iir & SDE_TRANSA_FIFO_UNDER)
8664281b
PZ		 1234 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1235 false))
1236 DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1237
1238 if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1239 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1240 false))
1241 DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1242}
1243
1244static void ivb_err_int_handler(struct drm_device *dev)
1245{
1246 struct drm_i915_private *dev_priv = dev->dev_private;
1247 u32 err_int = I915_READ(GEN7_ERR_INT);
1248
de032bf4
PZ		 1249 if (err_int & ERR_INT_POISON)
1250 DRM_ERROR("Poison interrupt\n");
1251
8664281b
PZ		 1252 if (err_int & ERR_INT_FIFO_UNDERRUN_A)
1253 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1254 DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1255
1256 if (err_int & ERR_INT_FIFO_UNDERRUN_B)
1257 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1258 DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1259
1260 if (err_int & ERR_INT_FIFO_UNDERRUN_C)
1261 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_C, false))
1262 DRM_DEBUG_DRIVER("Pipe C FIFO underrun\n");
1263
1264 I915_WRITE(GEN7_ERR_INT, err_int);
1265}
1266
1267static void cpt_serr_int_handler(struct drm_device *dev)
1268{
1269 struct drm_i915_private *dev_priv = dev->dev_private;
1270 u32 serr_int = I915_READ(SERR_INT);
1271
de032bf4
PZ		 1272 if (serr_int & SERR_INT_POISON)
1273 DRM_ERROR("PCH poison interrupt\n");
1274
8664281b
PZ		 1275 if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1276 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1277 false))
1278 DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1279
1280 if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1281 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1282 false))
1283 DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1284
1285 if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1286 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
1287 false))
1288 DRM_DEBUG_DRIVER("PCH transcoder C FIFO underrun\n");
1289
1290 I915_WRITE(SERR_INT, serr_int);
776ad806
JB		 1291}
1292
23e81d69
AJ		 1293static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1294{
1295 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1296 int pipe;
b543fb04 1297 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
23e81d69 1298
91d131d2
DV		 1299 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1300
cfc33bf7
VS		 1301 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1302 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1303 SDE_AUDIO_POWER_SHIFT_CPT);
1304 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1305 port_name(port));
1306 }
23e81d69
AJ		 1307
1308 if (pch_iir & SDE_AUX_MASK_CPT)
ce99c256 1309 dp_aux_irq_handler(dev);
23e81d69
AJ		 1310
1311 if (pch_iir & SDE_GMBUS_CPT)
515ac2bb 1312 gmbus_irq_handler(dev);
23e81d69
AJ		 1313
1314 if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1315 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1316
1317 if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1318 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1319
1320 if (pch_iir & SDE_FDI_MASK_CPT)
1321 for_each_pipe(pipe)
1322 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1323 pipe_name(pipe),
1324 I915_READ(FDI_RX_IIR(pipe)));
8664281b
PZ		 1325
1326 if (pch_iir & SDE_ERROR_CPT)
1327 cpt_serr_int_handler(dev);
23e81d69
AJ		 1328}
1329
c008bc6e
PZ		 1330static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
1331{
1332 struct drm_i915_private *dev_priv = dev->dev_private;
1333
1334 if (de_iir & DE_AUX_CHANNEL_A)
1335 dp_aux_irq_handler(dev);
1336
1337 if (de_iir & DE_GSE)
1338 intel_opregion_asle_intr(dev);
1339
1340 if (de_iir & DE_PIPEA_VBLANK)
1341 drm_handle_vblank(dev, 0);
1342
1343 if (de_iir & DE_PIPEB_VBLANK)
1344 drm_handle_vblank(dev, 1);
1345
1346 if (de_iir & DE_POISON)
1347 DRM_ERROR("Poison interrupt\n");
1348
1349 if (de_iir & DE_PIPEA_FIFO_UNDERRUN)
1350 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1351 DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1352
1353 if (de_iir & DE_PIPEB_FIFO_UNDERRUN)
1354 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1355 DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1356
1357 if (de_iir & DE_PLANEA_FLIP_DONE) {
1358 intel_prepare_page_flip(dev, 0);
1359 intel_finish_page_flip_plane(dev, 0);
1360 }
1361
1362 if (de_iir & DE_PLANEB_FLIP_DONE) {
1363 intel_prepare_page_flip(dev, 1);
1364 intel_finish_page_flip_plane(dev, 1);
1365 }
1366
1367 /* check event from PCH */
1368 if (de_iir & DE_PCH_EVENT) {
1369 u32 pch_iir = I915_READ(SDEIIR);
1370
1371 if (HAS_PCH_CPT(dev))
1372 cpt_irq_handler(dev, pch_iir);
1373 else
1374 ibx_irq_handler(dev, pch_iir);
1375
1376 /* should clear PCH hotplug event before clear CPU irq */
1377 I915_WRITE(SDEIIR, pch_iir);
1378 }
1379
1380 if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
1381 ironlake_rps_change_irq_handler(dev);
1382}
1383
9719fb98
PZ		 1384static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
1385{
1386 struct drm_i915_private *dev_priv = dev->dev_private;
1387 int i;
1388
1389 if (de_iir & DE_ERR_INT_IVB)
1390 ivb_err_int_handler(dev);
1391
1392 if (de_iir & DE_AUX_CHANNEL_A_IVB)
1393 dp_aux_irq_handler(dev);
1394
1395 if (de_iir & DE_GSE_IVB)
1396 intel_opregion_asle_intr(dev);
1397
1398 for (i = 0; i < 3; i++) {
1399 if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i)))
1400 drm_handle_vblank(dev, i);
1401 if (de_iir & (DE_PLANEA_FLIP_DONE_IVB << (5 * i))) {
1402 intel_prepare_page_flip(dev, i);
1403 intel_finish_page_flip_plane(dev, i);
1404 }
1405 }
1406
1407 /* check event from PCH */
1408 if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
1409 u32 pch_iir = I915_READ(SDEIIR);
1410
1411 cpt_irq_handler(dev, pch_iir);
1412
1413 /* clear PCH hotplug event before clear CPU irq */
1414 I915_WRITE(SDEIIR, pch_iir);
1415 }
1416}
1417
f1af8fc1 1418static irqreturn_t ironlake_irq_handler(int irq, void *arg)
b1f14ad0
JB		 1419{
1420 struct drm_device *dev = (struct drm_device *) arg;
1421 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
f1af8fc1 1422 u32 de_iir, gt_iir, de_ier, sde_ier = 0;
0e43406b 1423 irqreturn_t ret = IRQ_NONE;
333a8204 1424 bool err_int_reenable = false;
b1f14ad0
JB		 1425
1426 atomic_inc(&dev_priv->irq_received);
1427
8664281b
PZ		 1428 /* We get interrupts on unclaimed registers, so check for this before we
1429 * do any I915_{READ,WRITE}. */
907b28c5 1430 intel_uncore_check_errors(dev);
8664281b 1431
b1f14ad0
JB		 1432 /* disable master interrupt before clearing iir */
1433 de_ier = I915_READ(DEIER);
1434 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
23a78516 1435 POSTING_READ(DEIER);
b1f14ad0 1436
44498aea
PZ		 1437 /* Disable south interrupts. We'll only write to SDEIIR once, so further
1438 * interrupts will will be stored on its back queue, and then we'll be
1439 * able to process them after we restore SDEIER (as soon as we restore
1440 * it, we'll get an interrupt if SDEIIR still has something to process
1441 * due to its back queue). */
ab5c608b
BW		 1442 if (!HAS_PCH_NOP(dev)) {
1443 sde_ier = I915_READ(SDEIER);
1444 I915_WRITE(SDEIER, 0);
1445 POSTING_READ(SDEIER);
1446 }
44498aea 1447
8664281b
PZ		 1448 /* On Haswell, also mask ERR_INT because we don't want to risk
1449 * generating "unclaimed register" interrupts from inside the interrupt
1450 * handler. */
4bc9d430
DV		 1451 if (IS_HASWELL(dev)) {
1452 spin_lock(&dev_priv->irq_lock);
333a8204
PZ		 1453 err_int_reenable = ~dev_priv->irq_mask & DE_ERR_INT_IVB;
1454 if (err_int_reenable)
1455 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
4bc9d430
DV		 1456 spin_unlock(&dev_priv->irq_lock);
1457 }
8664281b 1458
b1f14ad0 1459 gt_iir = I915_READ(GTIIR);
0e43406b 1460 if (gt_iir) {
d8fc8a47 1461 if (INTEL_INFO(dev)->gen >= 6)
f1af8fc1 1462 snb_gt_irq_handler(dev, dev_priv, gt_iir);
d8fc8a47
PZ		 1463 else
1464 ilk_gt_irq_handler(dev, dev_priv, gt_iir);
0e43406b
CW		 1465 I915_WRITE(GTIIR, gt_iir);
1466 ret = IRQ_HANDLED;
b1f14ad0
JB		 1467 }
1468
0e43406b
CW		 1469 de_iir = I915_READ(DEIIR);
1470 if (de_iir) {
f1af8fc1
PZ		 1471 if (INTEL_INFO(dev)->gen >= 7)
1472 ivb_display_irq_handler(dev, de_iir);
1473 else
1474 ilk_display_irq_handler(dev, de_iir);
0e43406b
CW		 1475 I915_WRITE(DEIIR, de_iir);
1476 ret = IRQ_HANDLED;
b1f14ad0
JB		 1477 }
1478
f1af8fc1
PZ		 1479 if (INTEL_INFO(dev)->gen >= 6) {
1480 u32 pm_iir = I915_READ(GEN6_PMIIR);
1481 if (pm_iir) {
1403c0d4 1482 gen6_rps_irq_handler(dev_priv, pm_iir);
f1af8fc1
PZ		 1483 I915_WRITE(GEN6_PMIIR, pm_iir);
1484 ret = IRQ_HANDLED;
1485 }
0e43406b 1486 }
b1f14ad0 1487
333a8204 1488 if (err_int_reenable) {
4bc9d430
DV		 1489 spin_lock(&dev_priv->irq_lock);
1490 if (ivb_can_enable_err_int(dev))
1491 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
1492 spin_unlock(&dev_priv->irq_lock);
1493 }
8664281b 1494
b1f14ad0
JB		 1495 I915_WRITE(DEIER, de_ier);
1496 POSTING_READ(DEIER);
ab5c608b
BW		 1497 if (!HAS_PCH_NOP(dev)) {
1498 I915_WRITE(SDEIER, sde_ier);
1499 POSTING_READ(SDEIER);
1500 }
b1f14ad0
JB		 1501
1502 return ret;
1503}
1504
8a905236
JB		 1505/**
1506 * i915_error_work_func - do process context error handling work
1507 * @work: work struct
1508 *
1509 * Fire an error uevent so userspace can see that a hang or error
1510 * was detected.
1511 */
1512static void i915_error_work_func(struct work_struct *work)
1513{
1f83fee0
DV		 1514 struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
1515 work);
1516 drm_i915_private_t *dev_priv = container_of(error, drm_i915_private_t,
1517 gpu_error);
8a905236 1518 struct drm_device *dev = dev_priv->dev;
f69061be 1519 struct intel_ring_buffer *ring;
cce723ed
BW		 1520 char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
1521 char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
1522 char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
f69061be 1523 int i, ret;
8a905236 1524
f316a42c
BG		 1525 kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
1526
7db0ba24
DV		 1527 /*
1528 * Note that there's only one work item which does gpu resets, so we
1529 * need not worry about concurrent gpu resets potentially incrementing
1530 * error->reset_counter twice. We only need to take care of another
1531 * racing irq/hangcheck declaring the gpu dead for a second time. A
1532 * quick check for that is good enough: schedule_work ensures the
1533 * correct ordering between hang detection and this work item, and since
1534 * the reset in-progress bit is only ever set by code outside of this
1535 * work we don't need to worry about any other races.
1536 */
1537 if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
f803aa55 1538 DRM_DEBUG_DRIVER("resetting chip\n");
7db0ba24
DV		 1539 kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE,
1540 reset_event);
1f83fee0 1541
f69061be
DV		 1542 ret = i915_reset(dev);
1543
1544 if (ret == 0) {
1545 /*
1546 * After all the gem state is reset, increment the reset
1547 * counter and wake up everyone waiting for the reset to
1548 * complete.
1549 *
1550 * Since unlock operations are a one-sided barrier only,
1551 * we need to insert a barrier here to order any seqno
1552 * updates before
1553 * the counter increment.
1554 */
1555 smp_mb__before_atomic_inc();
1556 atomic_inc(&dev_priv->gpu_error.reset_counter);
1557
1558 kobject_uevent_env(&dev->primary->kdev.kobj,
1559 KOBJ_CHANGE, reset_done_event);
1f83fee0
DV		 1560 } else {
1561 atomic_set(&error->reset_counter, I915_WEDGED);
f316a42c 1562 }
1f83fee0 1563
f69061be
DV		 1564 for_each_ring(ring, dev_priv, i)
1565 wake_up_all(&ring->irq_queue);
1566
96a02917
VS		 1567 intel_display_handle_reset(dev);
1568
1f83fee0 1569 wake_up_all(&dev_priv->gpu_error.reset_queue);
f316a42c 1570 }
8a905236
JB		 1571}
1572
35aed2e6 1573static void i915_report_and_clear_eir(struct drm_device *dev)
8a905236
JB		 1574{
1575 struct drm_i915_private *dev_priv = dev->dev_private;
bd9854f9 1576 uint32_t instdone[I915_NUM_INSTDONE_REG];
8a905236 1577 u32 eir = I915_READ(EIR);
050ee91f 1578 int pipe, i;
8a905236 1579
35aed2e6
CW		 1580 if (!eir)
1581 return;
8a905236 1582
a70491cc 1583 pr_err("render error detected, EIR: 0x%08x\n", eir);
8a905236 1584
bd9854f9
BW		 1585 i915_get_extra_instdone(dev, instdone);
1586
8a905236
JB		 1587 if (IS_G4X(dev)) {
1588 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
1589 u32 ipeir = I915_READ(IPEIR_I965);
1590
a70491cc
JP		 1591 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1592 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
050ee91f
BW		 1593 for (i = 0; i < ARRAY_SIZE(instdone); i++)
1594 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
a70491cc 1595 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
a70491cc 1596 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
8a905236 1597 I915_WRITE(IPEIR_I965, ipeir);
3143a2bf 1598 POSTING_READ(IPEIR_I965);
8a905236
JB		 1599 }
1600 if (eir & GM45_ERROR_PAGE_TABLE) {
1601 u32 pgtbl_err = I915_READ(PGTBL_ER);
a70491cc
JP		 1602 pr_err("page table error\n");
1603 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
8a905236 1604 I915_WRITE(PGTBL_ER, pgtbl_err);
3143a2bf 1605 POSTING_READ(PGTBL_ER);
8a905236
JB		 1606 }
1607 }
1608
a6c45cf0 1609 if (!IS_GEN2(dev)) {
8a905236
JB		 1610 if (eir & I915_ERROR_PAGE_TABLE) {
1611 u32 pgtbl_err = I915_READ(PGTBL_ER);
a70491cc
JP		 1612 pr_err("page table error\n");
1613 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
8a905236 1614 I915_WRITE(PGTBL_ER, pgtbl_err);
3143a2bf 1615 POSTING_READ(PGTBL_ER);
8a905236
JB		 1616 }
1617 }
1618
1619 if (eir & I915_ERROR_MEMORY_REFRESH) {
a70491cc 1620 pr_err("memory refresh error:\n");
9db4a9c7 1621 for_each_pipe(pipe)
a70491cc 1622 pr_err("pipe %c stat: 0x%08x\n",
9db4a9c7 1623 pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
8a905236
JB		 1624 /* pipestat has already been acked */
1625 }
1626 if (eir & I915_ERROR_INSTRUCTION) {
a70491cc
JP		 1627 pr_err("instruction error\n");
1628 pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM));
050ee91f
BW		 1629 for (i = 0; i < ARRAY_SIZE(instdone); i++)
1630 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
a6c45cf0 1631 if (INTEL_INFO(dev)->gen < 4) {
8a905236
JB		 1632 u32 ipeir = I915_READ(IPEIR);
1633
a70491cc
JP		 1634 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR));
1635 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR));
a70491cc 1636 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD));
8a905236 1637 I915_WRITE(IPEIR, ipeir);
3143a2bf 1638 POSTING_READ(IPEIR);
8a905236
JB		 1639 } else {
1640 u32 ipeir = I915_READ(IPEIR_I965);
1641
a70491cc
JP		 1642 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1643 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
a70491cc 1644 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
a70491cc 1645 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
8a905236 1646 I915_WRITE(IPEIR_I965, ipeir);
3143a2bf 1647 POSTING_READ(IPEIR_I965);
8a905236
JB		 1648 }
1649 }
1650
1651 I915_WRITE(EIR, eir);
3143a2bf 1652 POSTING_READ(EIR);
8a905236
JB		 1653 eir = I915_READ(EIR);
1654 if (eir) {
1655 /*
1656 * some errors might have become stuck,
1657 * mask them.
1658 */
1659 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
1660 I915_WRITE(EMR, I915_READ(EMR) | eir);
1661 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
1662 }
35aed2e6
CW		 1663}
1664
1665/**
1666 * i915_handle_error - handle an error interrupt
1667 * @dev: drm device
1668 *
1669 * Do some basic checking of regsiter state at error interrupt time and
1670 * dump it to the syslog. Also call i915_capture_error_state() to make
1671 * sure we get a record and make it available in debugfs. Fire a uevent
1672 * so userspace knows something bad happened (should trigger collection
1673 * of a ring dump etc.).
1674 */
527f9e90 1675void i915_handle_error(struct drm_device *dev, bool wedged)
35aed2e6
CW		 1676{
1677 struct drm_i915_private *dev_priv = dev->dev_private;
b4519513
CW		 1678 struct intel_ring_buffer *ring;
1679 int i;
35aed2e6
CW		 1680
1681 i915_capture_error_state(dev);
1682 i915_report_and_clear_eir(dev);
8a905236 1683
ba1234d1 1684 if (wedged) {
f69061be
DV		 1685 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
1686 &dev_priv->gpu_error.reset_counter);
ba1234d1 1687
11ed50ec 1688 /*
1f83fee0
DV		 1689 * Wakeup waiting processes so that the reset work item
1690 * doesn't deadlock trying to grab various locks.
11ed50ec 1691 */
b4519513
CW		 1692 for_each_ring(ring, dev_priv, i)
1693 wake_up_all(&ring->irq_queue);
11ed50ec
BG		 1694 }
1695
99584db3 1696 queue_work(dev_priv->wq, &dev_priv->gpu_error.work);
8a905236
JB		 1697}
1698
21ad8330 1699static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
4e5359cd
SF		 1700{
1701 drm_i915_private_t *dev_priv = dev->dev_private;
1702 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1703 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
05394f39 1704 struct drm_i915_gem_object *obj;
4e5359cd
SF		 1705 struct intel_unpin_work *work;
1706 unsigned long flags;
1707 bool stall_detected;
1708
1709 /* Ignore early vblank irqs */
1710 if (intel_crtc == NULL)
1711 return;
1712
1713 spin_lock_irqsave(&dev->event_lock, flags);
1714 work = intel_crtc->unpin_work;
1715
e7d841ca
CW		 1716 if (work == NULL ||
1717 atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
1718 !work->enable_stall_check) {
4e5359cd
SF		 1719 /* Either the pending flip IRQ arrived, or we're too early. Don't check */
1720 spin_unlock_irqrestore(&dev->event_lock, flags);
1721 return;
1722 }
1723
1724 /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
05394f39 1725 obj = work->pending_flip_obj;
a6c45cf0 1726 if (INTEL_INFO(dev)->gen >= 4) {
9db4a9c7 1727 int dspsurf = DSPSURF(intel_crtc->plane);
446f2545 1728 stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
f343c5f6 1729 i915_gem_obj_ggtt_offset(obj);
4e5359cd 1730 } else {
9db4a9c7 1731 int dspaddr = DSPADDR(intel_crtc->plane);
f343c5f6 1732 stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
01f2c773 1733 crtc->y * crtc->fb->pitches[0] +
4e5359cd
SF		 1734 crtc->x * crtc->fb->bits_per_pixel/8);
1735 }
1736
1737 spin_unlock_irqrestore(&dev->event_lock, flags);
1738
1739 if (stall_detected) {
1740 DRM_DEBUG_DRIVER("Pageflip stall detected\n");
1741 intel_prepare_page_flip(dev, intel_crtc->plane);
1742 }
1743}
1744
42f52ef8
KP		 1745/* Called from drm generic code, passed 'crtc' which
1746 * we use as a pipe index
1747 */
f71d4af4 1748static int i915_enable_vblank(struct drm_device *dev, int pipe)
0a3e67a4
JB		 1749{
1750 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
e9d21d7f 1751 unsigned long irqflags;
71e0ffa5 1752
5eddb70b 1753 if (!i915_pipe_enabled(dev, pipe))
71e0ffa5 1754 return -EINVAL;
0a3e67a4 1755
1ec14ad3 1756 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
f796cf8f 1757 if (INTEL_INFO(dev)->gen >= 4)
7c463586
KP		 1758 i915_enable_pipestat(dev_priv, pipe,
1759 PIPE_START_VBLANK_INTERRUPT_ENABLE);
e9d21d7f 1760 else
7c463586
KP		 1761 i915_enable_pipestat(dev_priv, pipe,
1762 PIPE_VBLANK_INTERRUPT_ENABLE);
8692d00e
CW		 1763
1764 /* maintain vblank delivery even in deep C-states */
1765 if (dev_priv->info->gen == 3)
6b26c86d 1766 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
1ec14ad3 1767 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
8692d00e 1768
0a3e67a4
JB		 1769 return 0;
1770}
1771
f71d4af4 1772static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
f796cf8f
JB		 1773{
1774 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1775 unsigned long irqflags;
b518421f
PZ		 1776 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
1777 DE_PIPE_VBLANK_ILK(pipe);
f796cf8f
JB		 1778
1779 if (!i915_pipe_enabled(dev, pipe))
1780 return -EINVAL;
1781
1782 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
b518421f 1783 ironlake_enable_display_irq(dev_priv, bit);
b1f14ad0
JB		 1784 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1785
1786 return 0;
1787}
1788
7e231dbe
JB		 1789static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
1790{
1791 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1792 unsigned long irqflags;
31acc7f5 1793 u32 imr;
7e231dbe
JB		 1794
1795 if (!i915_pipe_enabled(dev, pipe))
1796 return -EINVAL;
1797
1798 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
7e231dbe 1799 imr = I915_READ(VLV_IMR);
31acc7f5 1800 if (pipe == 0)
7e231dbe 1801 imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
31acc7f5 1802 else
7e231dbe 1803 imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
7e231dbe 1804 I915_WRITE(VLV_IMR, imr);
31acc7f5
JB		 1805 i915_enable_pipestat(dev_priv, pipe,
1806 PIPE_START_VBLANK_INTERRUPT_ENABLE);
7e231dbe
JB		 1807 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1808
1809 return 0;
1810}
1811
42f52ef8
KP		 1812/* Called from drm generic code, passed 'crtc' which
1813 * we use as a pipe index
1814 */
f71d4af4 1815static void i915_disable_vblank(struct drm_device *dev, int pipe)
0a3e67a4
JB		 1816{
1817 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
e9d21d7f 1818 unsigned long irqflags;
0a3e67a4 1819
1ec14ad3 1820 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
8692d00e 1821 if (dev_priv->info->gen == 3)
6b26c86d 1822 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
8692d00e 1823
f796cf8f
JB		 1824 i915_disable_pipestat(dev_priv, pipe,
1825 PIPE_VBLANK_INTERRUPT_ENABLE |
1826 PIPE_START_VBLANK_INTERRUPT_ENABLE);
1827 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1828}
1829
f71d4af4 1830static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
f796cf8f
JB		 1831{
1832 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1833 unsigned long irqflags;
b518421f
PZ		 1834 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
1835 DE_PIPE_VBLANK_ILK(pipe);
f796cf8f
JB		 1836
1837 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
b518421f 1838 ironlake_disable_display_irq(dev_priv, bit);
b1f14ad0
JB		 1839 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1840}
1841
7e231dbe
JB		 1842static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
1843{
1844 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1845 unsigned long irqflags;
31acc7f5 1846 u32 imr;
7e231dbe
JB		 1847
1848 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
31acc7f5
JB		 1849 i915_disable_pipestat(dev_priv, pipe,
1850 PIPE_START_VBLANK_INTERRUPT_ENABLE);
7e231dbe 1851 imr = I915_READ(VLV_IMR);
31acc7f5 1852 if (pipe == 0)
7e231dbe 1853 imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
31acc7f5 1854 else
7e231dbe 1855 imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
7e231dbe 1856 I915_WRITE(VLV_IMR, imr);
7e231dbe
JB		 1857 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1858}
1859
893eead0
CW		 1860static u32
1861ring_last_seqno(struct intel_ring_buffer *ring)
852835f3 1862{
893eead0
CW		 1863 return list_entry(ring->request_list.prev,
1864 struct drm_i915_gem_request, list)->seqno;
1865}
1866
9107e9d2
CW		 1867static bool
1868ring_idle(struct intel_ring_buffer *ring, u32 seqno)
1869{
1870 return (list_empty(&ring->request_list) ||
1871 i915_seqno_passed(seqno, ring_last_seqno(ring)));
f65d9421
BG		 1872}
1873
6274f212
CW		 1874static struct intel_ring_buffer *
1875semaphore_waits_for(struct intel_ring_buffer *ring, u32 *seqno)
a24a11e6
CW		 1876{
1877 struct drm_i915_private *dev_priv = ring->dev->dev_private;
6274f212 1878 u32 cmd, ipehr, acthd, acthd_min;
a24a11e6
CW		 1879
1880 ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
1881 if ((ipehr & ~(0x3 << 16)) !=
1882 (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER))
6274f212 1883 return NULL;
a24a11e6
CW		 1884
1885 /* ACTHD is likely pointing to the dword after the actual command,
1886 * so scan backwards until we find the MBOX.
1887 */
6274f212 1888 acthd = intel_ring_get_active_head(ring) & HEAD_ADDR;
a24a11e6
CW		 1889 acthd_min = max((int)acthd - 3 * 4, 0);
1890 do {
1891 cmd = ioread32(ring->virtual_start + acthd);
1892 if (cmd == ipehr)
1893 break;
1894
1895 acthd -= 4;
1896 if (acthd < acthd_min)
6274f212 1897 return NULL;
a24a11e6
CW		 1898 } while (1);
1899
6274f212
CW		 1900 *seqno = ioread32(ring->virtual_start+acthd+4)+1;
1901 return &dev_priv->ring[(ring->id + (((ipehr >> 17) & 1) + 1)) % 3];
a24a11e6
CW		 1902}
1903
6274f212
CW		 1904static int semaphore_passed(struct intel_ring_buffer *ring)
1905{
1906 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1907 struct intel_ring_buffer *signaller;
1908 u32 seqno, ctl;
1909
1910 ring->hangcheck.deadlock = true;
1911
1912 signaller = semaphore_waits_for(ring, &seqno);
1913 if (signaller == NULL || signaller->hangcheck.deadlock)
1914 return -1;
1915
1916 /* cursory check for an unkickable deadlock */
1917 ctl = I915_READ_CTL(signaller);
1918 if (ctl & RING_WAIT_SEMAPHORE && semaphore_passed(signaller) < 0)
1919 return -1;
1920
1921 return i915_seqno_passed(signaller->get_seqno(signaller, false), seqno);
1922}
1923
1924static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
1925{
1926 struct intel_ring_buffer *ring;
1927 int i;
1928
1929 for_each_ring(ring, dev_priv, i)
1930 ring->hangcheck.deadlock = false;
1931}
1932
ad8beaea
MK		 1933static enum intel_ring_hangcheck_action
1934ring_stuck(struct intel_ring_buffer *ring, u32 acthd)
1ec14ad3
CW		 1935{
1936 struct drm_device *dev = ring->dev;
1937 struct drm_i915_private *dev_priv = dev->dev_private;
9107e9d2
CW		 1938 u32 tmp;
1939
6274f212 1940 if (ring->hangcheck.acthd != acthd)
f2f4d82f 1941 return HANGCHECK_ACTIVE;
6274f212 1942
9107e9d2 1943 if (IS_GEN2(dev))
f2f4d82f 1944 return HANGCHECK_HUNG;
9107e9d2
CW		 1945
1946 /* Is the chip hanging on a WAIT_FOR_EVENT?
1947 * If so we can simply poke the RB_WAIT bit
1948 * and break the hang. This should work on
1949 * all but the second generation chipsets.
1950 */
1951 tmp = I915_READ_CTL(ring);
1ec14ad3
CW		 1952 if (tmp & RING_WAIT) {
1953 DRM_ERROR("Kicking stuck wait on %s\n",
1954 ring->name);
1955 I915_WRITE_CTL(ring, tmp);
f2f4d82f 1956 return HANGCHECK_KICK;
6274f212
CW		 1957 }
1958
1959 if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
1960 switch (semaphore_passed(ring)) {
1961 default:
f2f4d82f 1962 return HANGCHECK_HUNG;
6274f212
CW		 1963 case 1:
1964 DRM_ERROR("Kicking stuck semaphore on %s\n",
1965 ring->name);
1966 I915_WRITE_CTL(ring, tmp);
f2f4d82f 1967 return HANGCHECK_KICK;
6274f212 1968 case 0:
f2f4d82f 1969 return HANGCHECK_WAIT;
6274f212 1970 }
9107e9d2 1971 }
ed5cbb03 1972
f2f4d82f 1973 return HANGCHECK_HUNG;
ed5cbb03
MK		 1974}
1975
f65d9421
BG		 1976/**
1977 * This is called when the chip hasn't reported back with completed
05407ff8
MK		 1978 * batchbuffers in a long time. We keep track per ring seqno progress and
1979 * if there are no progress, hangcheck score for that ring is increased.
1980 * Further, acthd is inspected to see if the ring is stuck. On stuck case
1981 * we kick the ring. If we see no progress on three subsequent calls
1982 * we assume chip is wedged and try to fix it by resetting the chip.
f65d9421 1983 */
a658b5d2 1984static void i915_hangcheck_elapsed(unsigned long data)
f65d9421
BG		 1985{
1986 struct drm_device *dev = (struct drm_device *)data;
1987 drm_i915_private_t *dev_priv = dev->dev_private;
b4519513 1988 struct intel_ring_buffer *ring;
b4519513 1989 int i;
05407ff8 1990 int busy_count = 0, rings_hung = 0;
9107e9d2
CW		 1991 bool stuck[I915_NUM_RINGS] = { 0 };
1992#define BUSY 1
1993#define KICK 5
1994#define HUNG 20
1995#define FIRE 30
893eead0 1996
3e0dc6b0
BW		 1997 if (!i915_enable_hangcheck)
1998 return;
1999
b4519513 2000 for_each_ring(ring, dev_priv, i) {
05407ff8 2001 u32 seqno, acthd;
9107e9d2 2002 bool busy = true;
05407ff8 2003
6274f212
CW		 2004 semaphore_clear_deadlocks(dev_priv);
2005
05407ff8
MK		 2006 seqno = ring->get_seqno(ring, false);
2007 acthd = intel_ring_get_active_head(ring);
b4519513 2008
9107e9d2
CW		 2009 if (ring->hangcheck.seqno == seqno) {
2010 if (ring_idle(ring, seqno)) {
da661464
MK		 2011 ring->hangcheck.action = HANGCHECK_IDLE;
2012
9107e9d2
CW		 2013 if (waitqueue_active(&ring->irq_queue)) {
2014 /* Issue a wake-up to catch stuck h/w. */
2015 DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2016 ring->name);
2017 wake_up_all(&ring->irq_queue);
2018 ring->hangcheck.score += HUNG;
2019 } else
2020 busy = false;
05407ff8 2021 } else {
6274f212
CW		 2022 /* We always increment the hangcheck score
2023 * if the ring is busy and still processing
2024 * the same request, so that no single request
2025 * can run indefinitely (such as a chain of
2026 * batches). The only time we do not increment
2027 * the hangcheck score on this ring, if this
2028 * ring is in a legitimate wait for another
2029 * ring. In that case the waiting ring is a
2030 * victim and we want to be sure we catch the
2031 * right culprit. Then every time we do kick
2032 * the ring, add a small increment to the
2033 * score so that we can catch a batch that is
2034 * being repeatedly kicked and so responsible
2035 * for stalling the machine.
2036 */
ad8beaea
MK		 2037 ring->hangcheck.action = ring_stuck(ring,
2038 acthd);
2039
2040 switch (ring->hangcheck.action) {
da661464 2041 case HANGCHECK_IDLE:
f2f4d82f 2042 case HANGCHECK_WAIT:
6274f212 2043 break;
f2f4d82f 2044 case HANGCHECK_ACTIVE:
ea04cb31 2045 ring->hangcheck.score += BUSY;
6274f212 2046 break;
f2f4d82f 2047 case HANGCHECK_KICK:
ea04cb31 2048 ring->hangcheck.score += KICK;
6274f212 2049 break;
f2f4d82f 2050 case HANGCHECK_HUNG:
ea04cb31 2051 ring->hangcheck.score += HUNG;
6274f212
CW		 2052 stuck[i] = true;
2053 break;
2054 }
05407ff8 2055 }
9107e9d2 2056 } else {
da661464
MK		 2057 ring->hangcheck.action = HANGCHECK_ACTIVE;
2058
9107e9d2
CW		 2059 /* Gradually reduce the count so that we catch DoS
2060 * attempts across multiple batches.
2061 */
2062 if (ring->hangcheck.score > 0)
2063 ring->hangcheck.score--;
d1e61e7f
CW		 2064 }
2065
05407ff8
MK		 2066 ring->hangcheck.seqno = seqno;
2067 ring->hangcheck.acthd = acthd;
9107e9d2 2068 busy_count += busy;
893eead0 2069 }
b9201c14 2070
92cab734 2071 for_each_ring(ring, dev_priv, i) {
9107e9d2 2072 if (ring->hangcheck.score > FIRE) {
b8d88d1d
DV		 2073 DRM_INFO("%s on %s\n",
2074 stuck[i] ? "stuck" : "no progress",
2075 ring->name);
a43adf07 2076 rings_hung++;
92cab734
MK		 2077 }
2078 }
2079
05407ff8
MK		 2080 if (rings_hung)
2081 return i915_handle_error(dev, true);
f65d9421 2082
05407ff8
MK		 2083 if (busy_count)
2084 /* Reset timer case chip hangs without another request
2085 * being added */
10cd45b6
MK		 2086 i915_queue_hangcheck(dev);
2087}
2088
2089void i915_queue_hangcheck(struct drm_device *dev)
2090{
2091 struct drm_i915_private *dev_priv = dev->dev_private;
2092 if (!i915_enable_hangcheck)
2093 return;
2094
2095 mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2096 round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
f65d9421
BG		 2097}
2098
91738a95
PZ		 2099static void ibx_irq_preinstall(struct drm_device *dev)
2100{
2101 struct drm_i915_private *dev_priv = dev->dev_private;
2102
2103 if (HAS_PCH_NOP(dev))
2104 return;
2105
2106 /* south display irq */
2107 I915_WRITE(SDEIMR, 0xffffffff);
2108 /*
2109 * SDEIER is also touched by the interrupt handler to work around missed
2110 * PCH interrupts. Hence we can't update it after the interrupt handler
2111 * is enabled - instead we unconditionally enable all PCH interrupt
2112 * sources here, but then only unmask them as needed with SDEIMR.
2113 */
2114 I915_WRITE(SDEIER, 0xffffffff);
2115 POSTING_READ(SDEIER);
2116}
2117
d18ea1b5
DV		 2118static void gen5_gt_irq_preinstall(struct drm_device *dev)
2119{
2120 struct drm_i915_private *dev_priv = dev->dev_private;
2121
2122 /* and GT */
2123 I915_WRITE(GTIMR, 0xffffffff);
2124 I915_WRITE(GTIER, 0x0);
2125 POSTING_READ(GTIER);
2126
2127 if (INTEL_INFO(dev)->gen >= 6) {
2128 /* and PM */
2129 I915_WRITE(GEN6_PMIMR, 0xffffffff);
2130 I915_WRITE(GEN6_PMIER, 0x0);
2131 POSTING_READ(GEN6_PMIER);
2132 }
2133}
2134
1da177e4
LT		 2135/* drm_dma.h hooks
2136*/
f71d4af4 2137static void ironlake_irq_preinstall(struct drm_device *dev)
036a4a7d
ZW		 2138{
2139 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2140
4697995b
JB		 2141 atomic_set(&dev_priv->irq_received, 0);
2142
036a4a7d 2143 I915_WRITE(HWSTAM, 0xeffe);
bdfcdb63 2144
036a4a7d
ZW		 2145 I915_WRITE(DEIMR, 0xffffffff);
2146 I915_WRITE(DEIER, 0x0);
3143a2bf 2147 POSTING_READ(DEIER);
036a4a7d 2148
d18ea1b5 2149 gen5_gt_irq_preinstall(dev);
c650156a 2150
91738a95 2151 ibx_irq_preinstall(dev);
7d99163d
BW		 2152}
2153
7e231dbe
JB		 2154static void valleyview_irq_preinstall(struct drm_device *dev)
2155{
2156 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2157 int pipe;
2158
2159 atomic_set(&dev_priv->irq_received, 0);
2160
7e231dbe
JB		 2161 /* VLV magic */
2162 I915_WRITE(VLV_IMR, 0);
2163 I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2164 I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2165 I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2166
7e231dbe
JB		 2167 /* and GT */
2168 I915_WRITE(GTIIR, I915_READ(GTIIR));
2169 I915_WRITE(GTIIR, I915_READ(GTIIR));
d18ea1b5
DV		 2170
2171 gen5_gt_irq_preinstall(dev);
7e231dbe
JB		 2172
2173 I915_WRITE(DPINVGTT, 0xff);
2174
2175 I915_WRITE(PORT_HOTPLUG_EN, 0);
2176 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2177 for_each_pipe(pipe)
2178 I915_WRITE(PIPESTAT(pipe), 0xffff);
2179 I915_WRITE(VLV_IIR, 0xffffffff);
2180 I915_WRITE(VLV_IMR, 0xffffffff);
2181 I915_WRITE(VLV_IER, 0x0);
2182 POSTING_READ(VLV_IER);
2183}
2184
82a28bcf 2185static void ibx_hpd_irq_setup(struct drm_device *dev)
7fe0b973
KP		 2186{
2187 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
82a28bcf
DV		 2188 struct drm_mode_config *mode_config = &dev->mode_config;
2189 struct intel_encoder *intel_encoder;
fee884ed 2190 u32 hotplug_irqs, hotplug, enabled_irqs = 0;
82a28bcf
DV		 2191
2192 if (HAS_PCH_IBX(dev)) {
fee884ed 2193 hotplug_irqs = SDE_HOTPLUG_MASK;
82a28bcf 2194 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
cd569aed 2195 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
fee884ed 2196 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
82a28bcf 2197 } else {
fee884ed 2198 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
82a28bcf 2199 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
cd569aed 2200 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
fee884ed 2201 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
82a28bcf 2202 }
7fe0b973 2203
fee884ed 2204 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
82a28bcf
DV		 2205
2206 /*
2207 * Enable digital hotplug on the PCH, and configure the DP short pulse
2208 * duration to 2ms (which is the minimum in the Display Port spec)
2209 *
2210 * This register is the same on all known PCH chips.
2211 */
7fe0b973
KP		 2212 hotplug = I915_READ(PCH_PORT_HOTPLUG);
2213 hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
2214 hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
2215 hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
2216 hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
2217 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
2218}
2219
d46da437
PZ		 2220static void ibx_irq_postinstall(struct drm_device *dev)
2221{
2222 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
82a28bcf 2223 u32 mask;
e5868a31 2224
692a04cf
DV		 2225 if (HAS_PCH_NOP(dev))
2226 return;
2227
8664281b
PZ		 2228 if (HAS_PCH_IBX(dev)) {
2229 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_TRANSB_FIFO_UNDER |
de032bf4 2230 SDE_TRANSA_FIFO_UNDER | SDE_POISON;
8664281b
PZ		 2231 } else {
2232 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT | SDE_ERROR_CPT;
2233
2234 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2235 }
ab5c608b 2236
d46da437
PZ		 2237 I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2238 I915_WRITE(SDEIMR, ~mask);
d46da437
PZ		 2239}
2240
0a9a8c91
DV		 2241static void gen5_gt_irq_postinstall(struct drm_device *dev)
2242{
2243 struct drm_i915_private *dev_priv = dev->dev_private;
2244 u32 pm_irqs, gt_irqs;
2245
2246 pm_irqs = gt_irqs = 0;
2247
2248 dev_priv->gt_irq_mask = ~0;
2249 if (HAS_L3_GPU_CACHE(dev)) {
2250 /* L3 parity interrupt is always unmasked. */
35a85ac6
BW		 2251 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
2252 gt_irqs |= GT_PARITY_ERROR(dev);
0a9a8c91
DV		 2253 }
2254
2255 gt_irqs |= GT_RENDER_USER_INTERRUPT;
2256 if (IS_GEN5(dev)) {
2257 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
2258 ILK_BSD_USER_INTERRUPT;
2259 } else {
2260 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
2261 }
2262
2263 I915_WRITE(GTIIR, I915_READ(GTIIR));
2264 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
2265 I915_WRITE(GTIER, gt_irqs);
2266 POSTING_READ(GTIER);
2267
2268 if (INTEL_INFO(dev)->gen >= 6) {
2269 pm_irqs |= GEN6_PM_RPS_EVENTS;
2270
2271 if (HAS_VEBOX(dev))
2272 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
2273
605cd25b 2274 dev_priv->pm_irq_mask = 0xffffffff;
0a9a8c91 2275 I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
605cd25b 2276 I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
0a9a8c91
DV		 2277 I915_WRITE(GEN6_PMIER, pm_irqs);
2278 POSTING_READ(GEN6_PMIER);
2279 }
2280}
2281
f71d4af4 2282static int ironlake_irq_postinstall(struct drm_device *dev)
036a4a7d 2283{
4bc9d430 2284 unsigned long irqflags;
036a4a7d 2285 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
8e76f8dc
PZ		 2286 u32 display_mask, extra_mask;
2287
2288 if (INTEL_INFO(dev)->gen >= 7) {
2289 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
2290 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
2291 DE_PLANEB_FLIP_DONE_IVB |
2292 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB |
2293 DE_ERR_INT_IVB);
2294 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
2295 DE_PIPEA_VBLANK_IVB);
2296
2297 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
2298 } else {
2299 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
2300 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
2301 DE_AUX_CHANNEL_A | DE_PIPEB_FIFO_UNDERRUN |
2302 DE_PIPEA_FIFO_UNDERRUN | DE_POISON);
2303 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT;
2304 }
036a4a7d 2305
1ec14ad3 2306 dev_priv->irq_mask = ~display_mask;
036a4a7d
ZW		 2307
2308 /* should always can generate irq */
2309 I915_WRITE(DEIIR, I915_READ(DEIIR));
1ec14ad3 2310 I915_WRITE(DEIMR, dev_priv->irq_mask);
8e76f8dc 2311 I915_WRITE(DEIER, display_mask | extra_mask);
3143a2bf 2312 POSTING_READ(DEIER);
036a4a7d 2313
0a9a8c91 2314 gen5_gt_irq_postinstall(dev);
036a4a7d 2315
d46da437 2316 ibx_irq_postinstall(dev);
7fe0b973 2317
f97108d1 2318 if (IS_IRONLAKE_M(dev)) {
6005ce42
DV		 2319 /* Enable PCU event interrupts
2320 *
2321 * spinlocking not required here for correctness since interrupt
4bc9d430
DV		 2322 * setup is guaranteed to run in single-threaded context. But we
2323 * need it to make the assert_spin_locked happy. */
2324 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
f97108d1 2325 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
4bc9d430 2326 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
f97108d1
JB		 2327 }
2328
036a4a7d
ZW		 2329 return 0;
2330}
2331
7e231dbe
JB		 2332static int valleyview_irq_postinstall(struct drm_device *dev)
2333{
2334 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
7e231dbe 2335 u32 enable_mask;
31acc7f5 2336 u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV;
b79480ba 2337 unsigned long irqflags;
7e231dbe
JB		 2338
2339 enable_mask = I915_DISPLAY_PORT_INTERRUPT;
31acc7f5
JB		 2340 enable_mask |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2341 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2342 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
7e231dbe
JB		 2343 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2344
31acc7f5
JB		 2345 /*
2346 *Leave vblank interrupts masked initially. enable/disable will
2347 * toggle them based on usage.
2348 */
2349 dev_priv->irq_mask = (~enable_mask) |
2350 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2351 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
7e231dbe 2352
20afbda2
DV		 2353 I915_WRITE(PORT_HOTPLUG_EN, 0);
2354 POSTING_READ(PORT_HOTPLUG_EN);
2355
7e231dbe
JB		 2356 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
2357 I915_WRITE(VLV_IER, enable_mask);
2358 I915_WRITE(VLV_IIR, 0xffffffff);
2359 I915_WRITE(PIPESTAT(0), 0xffff);
2360 I915_WRITE(PIPESTAT(1), 0xffff);
2361 POSTING_READ(VLV_IER);
2362
b79480ba
DV		 2363 /* Interrupt setup is already guaranteed to be single-threaded, this is
2364 * just to make the assert_spin_locked check happy. */
2365 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
31acc7f5 2366 i915_enable_pipestat(dev_priv, 0, pipestat_enable);
515ac2bb 2367 i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
31acc7f5 2368 i915_enable_pipestat(dev_priv, 1, pipestat_enable);
b79480ba 2369 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
31acc7f5 2370
7e231dbe
JB		 2371 I915_WRITE(VLV_IIR, 0xffffffff);
2372 I915_WRITE(VLV_IIR, 0xffffffff);
2373
0a9a8c91 2374 gen5_gt_irq_postinstall(dev);
7e231dbe
JB		 2375
2376 /* ack & enable invalid PTE error interrupts */
2377#if 0 /* FIXME: add support to irq handler for checking these bits */
2378 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2379 I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
2380#endif
2381
2382 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
20afbda2
DV		 2383
2384 return 0;
2385}
2386
7e231dbe
JB		 2387static void valleyview_irq_uninstall(struct drm_device *dev)
2388{
2389 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2390 int pipe;
2391
2392 if (!dev_priv)
2393 return;
2394
ac4c16c5
EE		 2395 del_timer_sync(&dev_priv->hotplug_reenable_timer);
2396
7e231dbe
JB		 2397 for_each_pipe(pipe)
2398 I915_WRITE(PIPESTAT(pipe), 0xffff);
2399
2400 I915_WRITE(HWSTAM, 0xffffffff);
2401 I915_WRITE(PORT_HOTPLUG_EN, 0);
2402 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2403 for_each_pipe(pipe)
2404 I915_WRITE(PIPESTAT(pipe), 0xffff);
2405 I915_WRITE(VLV_IIR, 0xffffffff);
2406 I915_WRITE(VLV_IMR, 0xffffffff);
2407 I915_WRITE(VLV_IER, 0x0);
2408 POSTING_READ(VLV_IER);
2409}
2410
f71d4af4 2411static void ironlake_irq_uninstall(struct drm_device *dev)
036a4a7d
ZW		 2412{
2413 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
4697995b
JB		 2414
2415 if (!dev_priv)
2416 return;
2417
ac4c16c5
EE		 2418 del_timer_sync(&dev_priv->hotplug_reenable_timer);
2419
036a4a7d
ZW		 2420 I915_WRITE(HWSTAM, 0xffffffff);
2421
2422 I915_WRITE(DEIMR, 0xffffffff);
2423 I915_WRITE(DEIER, 0x0);
2424 I915_WRITE(DEIIR, I915_READ(DEIIR));
8664281b
PZ		 2425 if (IS_GEN7(dev))
2426 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
036a4a7d
ZW		 2427
2428 I915_WRITE(GTIMR, 0xffffffff);
2429 I915_WRITE(GTIER, 0x0);
2430 I915_WRITE(GTIIR, I915_READ(GTIIR));
192aac1f 2431
ab5c608b
BW		 2432 if (HAS_PCH_NOP(dev))
2433 return;
2434
192aac1f
KP		 2435 I915_WRITE(SDEIMR, 0xffffffff);
2436 I915_WRITE(SDEIER, 0x0);
2437 I915_WRITE(SDEIIR, I915_READ(SDEIIR));
8664281b
PZ		 2438 if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
2439 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
036a4a7d
ZW		 2440}
2441
a266c7d5 2442static void i8xx_irq_preinstall(struct drm_device * dev)
1da177e4
LT		 2443{
2444 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
9db4a9c7 2445 int pipe;
91e3738e 2446
a266c7d5 2447 atomic_set(&dev_priv->irq_received, 0);
5ca58282 2448
9db4a9c7
JB		 2449 for_each_pipe(pipe)
2450 I915_WRITE(PIPESTAT(pipe), 0);
a266c7d5
CW		 2451 I915_WRITE16(IMR, 0xffff);
2452 I915_WRITE16(IER, 0x0);
2453 POSTING_READ16(IER);
c2798b19
CW		 2454}
2455
2456static int i8xx_irq_postinstall(struct drm_device *dev)
2457{
2458 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2459
c2798b19
CW		 2460 I915_WRITE16(EMR,
2461 ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2462
2463 /* Unmask the interrupts that we always want on. */
2464 dev_priv->irq_mask =
2465 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2466 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2467 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2468 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2469 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2470 I915_WRITE16(IMR, dev_priv->irq_mask);
2471
2472 I915_WRITE16(IER,
2473 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2474 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2475 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2476 I915_USER_INTERRUPT);
2477 POSTING_READ16(IER);
2478
2479 return 0;
2480}
2481
90a72f87
VS		 2482/*
2483 * Returns true when a page flip has completed.
2484 */
2485static bool i8xx_handle_vblank(struct drm_device *dev,
2486 int pipe, u16 iir)
2487{
2488 drm_i915_private_t *dev_priv = dev->dev_private;
2489 u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(pipe);
2490
2491 if (!drm_handle_vblank(dev, pipe))
2492 return false;
2493
2494 if ((iir & flip_pending) == 0)
2495 return false;
2496
2497 intel_prepare_page_flip(dev, pipe);
2498
2499 /* We detect FlipDone by looking for the change in PendingFlip from '1'
2500 * to '0' on the following vblank, i.e. IIR has the Pendingflip
2501 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
2502 * the flip is completed (no longer pending). Since this doesn't raise
2503 * an interrupt per se, we watch for the change at vblank.
2504 */
2505 if (I915_READ16(ISR) & flip_pending)
2506 return false;
2507
2508 intel_finish_page_flip(dev, pipe);
2509
2510 return true;
2511}
2512
ff1f525e 2513static irqreturn_t i8xx_irq_handler(int irq, void *arg)
c2798b19
CW		 2514{
2515 struct drm_device *dev = (struct drm_device *) arg;
2516 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
c2798b19
CW		 2517 u16 iir, new_iir;
2518 u32 pipe_stats[2];
2519 unsigned long irqflags;
c2798b19
CW		 2520 int pipe;
2521 u16 flip_mask =
2522 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2523 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2524
2525 atomic_inc(&dev_priv->irq_received);
2526
2527 iir = I915_READ16(IIR);
2528 if (iir == 0)
2529 return IRQ_NONE;
2530
2531 while (iir & ~flip_mask) {
2532 /* Can't rely on pipestat interrupt bit in iir as it might
2533 * have been cleared after the pipestat interrupt was received.
2534 * It doesn't set the bit in iir again, but it still produces
2535 * interrupts (for non-MSI).
2536 */
2537 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2538 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2539 i915_handle_error(dev, false);
2540
2541 for_each_pipe(pipe) {
2542 int reg = PIPESTAT(pipe);
2543 pipe_stats[pipe] = I915_READ(reg);
2544
2545 /*
2546 * Clear the PIPE*STAT regs before the IIR
2547 */
2548 if (pipe_stats[pipe] & 0x8000ffff) {
2549 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2550 DRM_DEBUG_DRIVER("pipe %c underrun\n",
2551 pipe_name(pipe));
2552 I915_WRITE(reg, pipe_stats[pipe]);
c2798b19
CW		 2553 }
2554 }
2555 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2556
2557 I915_WRITE16(IIR, iir & ~flip_mask);
2558 new_iir = I915_READ16(IIR); /* Flush posted writes */
2559
d05c617e 2560 i915_update_dri1_breadcrumb(dev);
c2798b19
CW		 2561
2562 if (iir & I915_USER_INTERRUPT)
2563 notify_ring(dev, &dev_priv->ring[RCS]);
2564
2565 if (pipe_stats[0] & PIPE_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 2566 i8xx_handle_vblank(dev, 0, iir))
2567 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(0);
c2798b19
CW		 2568
2569 if (pipe_stats[1] & PIPE_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 2570 i8xx_handle_vblank(dev, 1, iir))
2571 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(1);
c2798b19
CW		 2572
2573 iir = new_iir;
2574 }
2575
2576 return IRQ_HANDLED;
2577}
2578
2579static void i8xx_irq_uninstall(struct drm_device * dev)
2580{
2581 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2582 int pipe;
2583
c2798b19
CW		 2584 for_each_pipe(pipe) {
2585 /* Clear enable bits; then clear status bits */
2586 I915_WRITE(PIPESTAT(pipe), 0);
2587 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2588 }
2589 I915_WRITE16(IMR, 0xffff);
2590 I915_WRITE16(IER, 0x0);
2591 I915_WRITE16(IIR, I915_READ16(IIR));
2592}
2593
a266c7d5
CW		 2594static void i915_irq_preinstall(struct drm_device * dev)
2595{
2596 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2597 int pipe;
2598
2599 atomic_set(&dev_priv->irq_received, 0);
2600
2601 if (I915_HAS_HOTPLUG(dev)) {
2602 I915_WRITE(PORT_HOTPLUG_EN, 0);
2603 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2604 }
2605
00d98ebd 2606 I915_WRITE16(HWSTAM, 0xeffe);
a266c7d5
CW		 2607 for_each_pipe(pipe)
2608 I915_WRITE(PIPESTAT(pipe), 0);
2609 I915_WRITE(IMR, 0xffffffff);
2610 I915_WRITE(IER, 0x0);
2611 POSTING_READ(IER);
2612}
2613
2614static int i915_irq_postinstall(struct drm_device *dev)
2615{
2616 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
38bde180 2617 u32 enable_mask;
a266c7d5 2618
38bde180
CW		 2619 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2620
2621 /* Unmask the interrupts that we always want on. */
2622 dev_priv->irq_mask =
2623 ~(I915_ASLE_INTERRUPT |
2624 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2625 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2626 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2627 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2628 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2629
2630 enable_mask =
2631 I915_ASLE_INTERRUPT |
2632 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2633 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2634 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2635 I915_USER_INTERRUPT;
2636
a266c7d5 2637 if (I915_HAS_HOTPLUG(dev)) {
20afbda2
DV		 2638 I915_WRITE(PORT_HOTPLUG_EN, 0);
2639 POSTING_READ(PORT_HOTPLUG_EN);
2640
a266c7d5
CW		 2641 /* Enable in IER... */
2642 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
2643 /* and unmask in IMR */
2644 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
2645 }
2646
a266c7d5
CW		 2647 I915_WRITE(IMR, dev_priv->irq_mask);
2648 I915_WRITE(IER, enable_mask);
2649 POSTING_READ(IER);
2650
f49e38dd 2651 i915_enable_asle_pipestat(dev);
20afbda2
DV		 2652
2653 return 0;
2654}
2655
90a72f87
VS		 2656/*
2657 * Returns true when a page flip has completed.
2658 */
2659static bool i915_handle_vblank(struct drm_device *dev,
2660 int plane, int pipe, u32 iir)
2661{
2662 drm_i915_private_t *dev_priv = dev->dev_private;
2663 u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
2664
2665 if (!drm_handle_vblank(dev, pipe))
2666 return false;
2667
2668 if ((iir & flip_pending) == 0)
2669 return false;
2670
2671 intel_prepare_page_flip(dev, plane);
2672
2673 /* We detect FlipDone by looking for the change in PendingFlip from '1'
2674 * to '0' on the following vblank, i.e. IIR has the Pendingflip
2675 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
2676 * the flip is completed (no longer pending). Since this doesn't raise
2677 * an interrupt per se, we watch for the change at vblank.
2678 */
2679 if (I915_READ(ISR) & flip_pending)
2680 return false;
2681
2682 intel_finish_page_flip(dev, pipe);
2683
2684 return true;
2685}
2686
ff1f525e 2687static irqreturn_t i915_irq_handler(int irq, void *arg)
a266c7d5
CW		 2688{
2689 struct drm_device *dev = (struct drm_device *) arg;
2690 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
8291ee90 2691 u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
a266c7d5 2692 unsigned long irqflags;
38bde180
CW		 2693 u32 flip_mask =
2694 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2695 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
38bde180 2696 int pipe, ret = IRQ_NONE;
a266c7d5
CW		 2697
2698 atomic_inc(&dev_priv->irq_received);
2699
2700 iir = I915_READ(IIR);
38bde180
CW		 2701 do {
2702 bool irq_received = (iir & ~flip_mask) != 0;
8291ee90 2703 bool blc_event = false;
a266c7d5
CW		 2704
2705 /* Can't rely on pipestat interrupt bit in iir as it might
2706 * have been cleared after the pipestat interrupt was received.
2707 * It doesn't set the bit in iir again, but it still produces
2708 * interrupts (for non-MSI).
2709 */
2710 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2711 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2712 i915_handle_error(dev, false);
2713
2714 for_each_pipe(pipe) {
2715 int reg = PIPESTAT(pipe);
2716 pipe_stats[pipe] = I915_READ(reg);
2717
38bde180 2718 /* Clear the PIPE*STAT regs before the IIR */
a266c7d5
CW		 2719 if (pipe_stats[pipe] & 0x8000ffff) {
2720 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2721 DRM_DEBUG_DRIVER("pipe %c underrun\n",
2722 pipe_name(pipe));
2723 I915_WRITE(reg, pipe_stats[pipe]);
38bde180 2724 irq_received = true;
a266c7d5
CW		 2725 }
2726 }
2727 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2728
2729 if (!irq_received)
2730 break;
2731
a266c7d5
CW		 2732 /* Consume port. Then clear IIR or we'll miss events */
2733 if ((I915_HAS_HOTPLUG(dev)) &&
2734 (iir & I915_DISPLAY_PORT_INTERRUPT)) {
2735 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
b543fb04 2736 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
a266c7d5
CW		 2737
2738 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2739 hotplug_status);
91d131d2
DV		 2740
2741 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
2742
a266c7d5 2743 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
38bde180 2744 POSTING_READ(PORT_HOTPLUG_STAT);
a266c7d5
CW		 2745 }
2746
38bde180 2747 I915_WRITE(IIR, iir & ~flip_mask);
a266c7d5
CW		 2748 new_iir = I915_READ(IIR); /* Flush posted writes */
2749
a266c7d5
CW		 2750 if (iir & I915_USER_INTERRUPT)
2751 notify_ring(dev, &dev_priv->ring[RCS]);
a266c7d5 2752
a266c7d5 2753 for_each_pipe(pipe) {
38bde180
CW		 2754 int plane = pipe;
2755 if (IS_MOBILE(dev))
2756 plane = !plane;
90a72f87 2757
8291ee90 2758 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 2759 i915_handle_vblank(dev, plane, pipe, iir))
2760 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
a266c7d5
CW		 2761
2762 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2763 blc_event = true;
2764 }
2765
a266c7d5
CW		 2766 if (blc_event || (iir & I915_ASLE_INTERRUPT))
2767 intel_opregion_asle_intr(dev);
2768
2769 /* With MSI, interrupts are only generated when iir
2770 * transitions from zero to nonzero. If another bit got
2771 * set while we were handling the existing iir bits, then
2772 * we would never get another interrupt.
2773 *
2774 * This is fine on non-MSI as well, as if we hit this path
2775 * we avoid exiting the interrupt handler only to generate
2776 * another one.
2777 *
2778 * Note that for MSI this could cause a stray interrupt report
2779 * if an interrupt landed in the time between writing IIR and
2780 * the posting read. This should be rare enough to never
2781 * trigger the 99% of 100,000 interrupts test for disabling
2782 * stray interrupts.
2783 */
38bde180 2784 ret = IRQ_HANDLED;
a266c7d5 2785 iir = new_iir;
38bde180 2786 } while (iir & ~flip_mask);
a266c7d5 2787
d05c617e 2788 i915_update_dri1_breadcrumb(dev);
8291ee90 2789
a266c7d5
CW		 2790 return ret;
2791}
2792
2793static void i915_irq_uninstall(struct drm_device * dev)
2794{
2795 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2796 int pipe;
2797
ac4c16c5
EE		 2798 del_timer_sync(&dev_priv->hotplug_reenable_timer);
2799
a266c7d5
CW		 2800 if (I915_HAS_HOTPLUG(dev)) {
2801 I915_WRITE(PORT_HOTPLUG_EN, 0);
2802 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2803 }
2804
00d98ebd 2805 I915_WRITE16(HWSTAM, 0xffff);
55b39755
CW		 2806 for_each_pipe(pipe) {
2807 /* Clear enable bits; then clear status bits */
a266c7d5 2808 I915_WRITE(PIPESTAT(pipe), 0);
55b39755
CW		 2809 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2810 }
a266c7d5
CW		 2811 I915_WRITE(IMR, 0xffffffff);
2812 I915_WRITE(IER, 0x0);
2813
a266c7d5
CW		 2814 I915_WRITE(IIR, I915_READ(IIR));
2815}
2816
2817static void i965_irq_preinstall(struct drm_device * dev)
2818{
2819 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2820 int pipe;
2821
2822 atomic_set(&dev_priv->irq_received, 0);
2823
adca4730
CW		 2824 I915_WRITE(PORT_HOTPLUG_EN, 0);
2825 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
a266c7d5
CW		 2826
2827 I915_WRITE(HWSTAM, 0xeffe);
2828 for_each_pipe(pipe)
2829 I915_WRITE(PIPESTAT(pipe), 0);
2830 I915_WRITE(IMR, 0xffffffff);
2831 I915_WRITE(IER, 0x0);
2832 POSTING_READ(IER);
2833}
2834
2835static int i965_irq_postinstall(struct drm_device *dev)
2836{
2837 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
bbba0a97 2838 u32 enable_mask;
a266c7d5 2839 u32 error_mask;
b79480ba 2840 unsigned long irqflags;
a266c7d5 2841
a266c7d5 2842 /* Unmask the interrupts that we always want on. */
bbba0a97 2843 dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
adca4730 2844 I915_DISPLAY_PORT_INTERRUPT |
bbba0a97
CW		 2845 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2846 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2847 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2848 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2849 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2850
2851 enable_mask = ~dev_priv->irq_mask;
21ad8330
VS		 2852 enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2853 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
bbba0a97
CW		 2854 enable_mask |= I915_USER_INTERRUPT;
2855
2856 if (IS_G4X(dev))
2857 enable_mask |= I915_BSD_USER_INTERRUPT;
a266c7d5 2858
b79480ba
DV		 2859 /* Interrupt setup is already guaranteed to be single-threaded, this is
2860 * just to make the assert_spin_locked check happy. */
2861 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
515ac2bb 2862 i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
b79480ba 2863 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
a266c7d5 2864
a266c7d5
CW		 2865 /*
2866 * Enable some error detection, note the instruction error mask
2867 * bit is reserved, so we leave it masked.
2868 */
2869 if (IS_G4X(dev)) {
2870 error_mask = ~(GM45_ERROR_PAGE_TABLE |
2871 GM45_ERROR_MEM_PRIV |
2872 GM45_ERROR_CP_PRIV |
2873 I915_ERROR_MEMORY_REFRESH);
2874 } else {
2875 error_mask = ~(I915_ERROR_PAGE_TABLE |
2876 I915_ERROR_MEMORY_REFRESH);
2877 }
2878 I915_WRITE(EMR, error_mask);
2879
2880 I915_WRITE(IMR, dev_priv->irq_mask);
2881 I915_WRITE(IER, enable_mask);
2882 POSTING_READ(IER);
2883
20afbda2
DV		 2884 I915_WRITE(PORT_HOTPLUG_EN, 0);
2885 POSTING_READ(PORT_HOTPLUG_EN);
2886
f49e38dd 2887 i915_enable_asle_pipestat(dev);
20afbda2
DV		 2888
2889 return 0;
2890}
2891
bac56d5b 2892static void i915_hpd_irq_setup(struct drm_device *dev)
20afbda2
DV		 2893{
2894 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
e5868a31 2895 struct drm_mode_config *mode_config = &dev->mode_config;
cd569aed 2896 struct intel_encoder *intel_encoder;
20afbda2
DV		 2897 u32 hotplug_en;
2898
b5ea2d56
DV		 2899 assert_spin_locked(&dev_priv->irq_lock);
2900
bac56d5b
EE		 2901 if (I915_HAS_HOTPLUG(dev)) {
2902 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
2903 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
2904 /* Note HDMI and DP share hotplug bits */
e5868a31 2905 /* enable bits are the same for all generations */
cd569aed
EE		 2906 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2907 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2908 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
bac56d5b
EE		 2909 /* Programming the CRT detection parameters tends
2910 to generate a spurious hotplug event about three
2911 seconds later. So just do it once.
2912 */
2913 if (IS_G4X(dev))
2914 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
85fc95ba 2915 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
bac56d5b 2916 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
a266c7d5 2917
bac56d5b
EE		 2918 /* Ignore TV since it's buggy */
2919 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
2920 }
a266c7d5
CW		 2921}
2922
ff1f525e 2923static irqreturn_t i965_irq_handler(int irq, void *arg)
a266c7d5
CW		 2924{
2925 struct drm_device *dev = (struct drm_device *) arg;
2926 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
a266c7d5
CW		 2927 u32 iir, new_iir;
2928 u32 pipe_stats[I915_MAX_PIPES];
a266c7d5
CW		 2929 unsigned long irqflags;
2930 int irq_received;
2931 int ret = IRQ_NONE, pipe;
21ad8330
VS		 2932 u32 flip_mask =
2933 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2934 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
a266c7d5
CW		 2935
2936 atomic_inc(&dev_priv->irq_received);
2937
2938 iir = I915_READ(IIR);
2939
a266c7d5 2940 for (;;) {
2c8ba29f
CW		 2941 bool blc_event = false;
2942
21ad8330 2943 irq_received = (iir & ~flip_mask) != 0;
a266c7d5
CW		 2944
2945 /* Can't rely on pipestat interrupt bit in iir as it might
2946 * have been cleared after the pipestat interrupt was received.
2947 * It doesn't set the bit in iir again, but it still produces
2948 * interrupts (for non-MSI).
2949 */
2950 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2951 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2952 i915_handle_error(dev, false);
2953
2954 for_each_pipe(pipe) {
2955 int reg = PIPESTAT(pipe);
2956 pipe_stats[pipe] = I915_READ(reg);
2957
2958 /*
2959 * Clear the PIPE*STAT regs before the IIR
2960 */
2961 if (pipe_stats[pipe] & 0x8000ffff) {
2962 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2963 DRM_DEBUG_DRIVER("pipe %c underrun\n",
2964 pipe_name(pipe));
2965 I915_WRITE(reg, pipe_stats[pipe]);
2966 irq_received = 1;
2967 }
2968 }
2969 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2970
2971 if (!irq_received)
2972 break;
2973
2974 ret = IRQ_HANDLED;
2975
2976 /* Consume port. Then clear IIR or we'll miss events */
adca4730 2977 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
a266c7d5 2978 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
b543fb04
EE		 2979 u32 hotplug_trigger = hotplug_status & (IS_G4X(dev) ?
2980 HOTPLUG_INT_STATUS_G4X :
4f7fd709 2981 HOTPLUG_INT_STATUS_I915);
a266c7d5
CW		 2982
2983 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2984 hotplug_status);
91d131d2
DV		 2985
2986 intel_hpd_irq_handler(dev, hotplug_trigger,
2987 IS_G4X(dev) ? hpd_status_gen4 : hpd_status_i915);
2988
a266c7d5
CW		 2989 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2990 I915_READ(PORT_HOTPLUG_STAT);
2991 }
2992
21ad8330 2993 I915_WRITE(IIR, iir & ~flip_mask);
a266c7d5
CW		 2994 new_iir = I915_READ(IIR); /* Flush posted writes */
2995
a266c7d5
CW		 2996 if (iir & I915_USER_INTERRUPT)
2997 notify_ring(dev, &dev_priv->ring[RCS]);
2998 if (iir & I915_BSD_USER_INTERRUPT)
2999 notify_ring(dev, &dev_priv->ring[VCS]);
3000
a266c7d5 3001 for_each_pipe(pipe) {
2c8ba29f 3002 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 3003 i915_handle_vblank(dev, pipe, pipe, iir))
3004 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
a266c7d5
CW		 3005
3006 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3007 blc_event = true;
3008 }
3009
3010
3011 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3012 intel_opregion_asle_intr(dev);
3013
515ac2bb
DV		 3014 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
3015 gmbus_irq_handler(dev);
3016
a266c7d5
CW		 3017 /* With MSI, interrupts are only generated when iir
3018 * transitions from zero to nonzero. If another bit got
3019 * set while we were handling the existing iir bits, then
3020 * we would never get another interrupt.
3021 *
3022 * This is fine on non-MSI as well, as if we hit this path
3023 * we avoid exiting the interrupt handler only to generate
3024 * another one.
3025 *
3026 * Note that for MSI this could cause a stray interrupt report
3027 * if an interrupt landed in the time between writing IIR and
3028 * the posting read. This should be rare enough to never
3029 * trigger the 99% of 100,000 interrupts test for disabling
3030 * stray interrupts.
3031 */
3032 iir = new_iir;
3033 }
3034
d05c617e 3035 i915_update_dri1_breadcrumb(dev);
2c8ba29f 3036
a266c7d5
CW		 3037 return ret;
3038}
3039
3040static void i965_irq_uninstall(struct drm_device * dev)
3041{
3042 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3043 int pipe;
3044
3045 if (!dev_priv)
3046 return;
3047
ac4c16c5
EE		 3048 del_timer_sync(&dev_priv->hotplug_reenable_timer);
3049
adca4730
CW		 3050 I915_WRITE(PORT_HOTPLUG_EN, 0);
3051 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
a266c7d5
CW		 3052
3053 I915_WRITE(HWSTAM, 0xffffffff);
3054 for_each_pipe(pipe)
3055 I915_WRITE(PIPESTAT(pipe), 0);
3056 I915_WRITE(IMR, 0xffffffff);
3057 I915_WRITE(IER, 0x0);
3058
3059 for_each_pipe(pipe)
3060 I915_WRITE(PIPESTAT(pipe),
3061 I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
3062 I915_WRITE(IIR, I915_READ(IIR));
3063}
3064
ac4c16c5
EE		 3065static void i915_reenable_hotplug_timer_func(unsigned long data)
3066{
3067 drm_i915_private_t *dev_priv = (drm_i915_private_t *)data;
3068 struct drm_device *dev = dev_priv->dev;
3069 struct drm_mode_config *mode_config = &dev->mode_config;
3070 unsigned long irqflags;
3071 int i;
3072
3073 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3074 for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
3075 struct drm_connector *connector;
3076
3077 if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
3078 continue;
3079
3080 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3081
3082 list_for_each_entry(connector, &mode_config->connector_list, head) {
3083 struct intel_connector *intel_connector = to_intel_connector(connector);
3084
3085 if (intel_connector->encoder->hpd_pin == i) {
3086 if (connector->polled != intel_connector->polled)
3087 DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
3088 drm_get_connector_name(connector));
3089 connector->polled = intel_connector->polled;
3090 if (!connector->polled)
3091 connector->polled = DRM_CONNECTOR_POLL_HPD;
3092 }
3093 }
3094 }
3095 if (dev_priv->display.hpd_irq_setup)
3096 dev_priv->display.hpd_irq_setup(dev);
3097 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3098}
3099
f71d4af4
JB		 3100void intel_irq_init(struct drm_device *dev)
3101{
8b2e326d
CW		 3102 struct drm_i915_private *dev_priv = dev->dev_private;
3103
3104 INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
99584db3 3105 INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
c6a828d3 3106 INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
a4da4fa4 3107 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
8b2e326d 3108
99584db3
DV		 3109 setup_timer(&dev_priv->gpu_error.hangcheck_timer,
3110 i915_hangcheck_elapsed,
61bac78e 3111 (unsigned long) dev);
ac4c16c5
EE		 3112 setup_timer(&dev_priv->hotplug_reenable_timer, i915_reenable_hotplug_timer_func,
3113 (unsigned long) dev_priv);
61bac78e 3114
97a19a24 3115 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
9ee32fea 3116
f71d4af4
JB		 3117 dev->driver->get_vblank_counter = i915_get_vblank_counter;
3118 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
7d4e146f 3119 if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
f71d4af4
JB		 3120 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
3121 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
3122 }
3123
c3613de9
KP		 3124 if (drm_core_check_feature(dev, DRIVER_MODESET))
3125 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
3126 else
3127 dev->driver->get_vblank_timestamp = NULL;
f71d4af4
JB		 3128 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
3129
7e231dbe
JB		 3130 if (IS_VALLEYVIEW(dev)) {
3131 dev->driver->irq_handler = valleyview_irq_handler;
3132 dev->driver->irq_preinstall = valleyview_irq_preinstall;
3133 dev->driver->irq_postinstall = valleyview_irq_postinstall;
3134 dev->driver->irq_uninstall = valleyview_irq_uninstall;
3135 dev->driver->enable_vblank = valleyview_enable_vblank;
3136 dev->driver->disable_vblank = valleyview_disable_vblank;
fa00abe0 3137 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
f71d4af4
JB		 3138 } else if (HAS_PCH_SPLIT(dev)) {
3139 dev->driver->irq_handler = ironlake_irq_handler;
3140 dev->driver->irq_preinstall = ironlake_irq_preinstall;
3141 dev->driver->irq_postinstall = ironlake_irq_postinstall;
3142 dev->driver->irq_uninstall = ironlake_irq_uninstall;
3143 dev->driver->enable_vblank = ironlake_enable_vblank;
3144 dev->driver->disable_vblank = ironlake_disable_vblank;
82a28bcf 3145 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
f71d4af4 3146 } else {
c2798b19
CW		 3147 if (INTEL_INFO(dev)->gen == 2) {
3148 dev->driver->irq_preinstall = i8xx_irq_preinstall;
3149 dev->driver->irq_postinstall = i8xx_irq_postinstall;
3150 dev->driver->irq_handler = i8xx_irq_handler;
3151 dev->driver->irq_uninstall = i8xx_irq_uninstall;
a266c7d5
CW		 3152 } else if (INTEL_INFO(dev)->gen == 3) {
3153 dev->driver->irq_preinstall = i915_irq_preinstall;
3154 dev->driver->irq_postinstall = i915_irq_postinstall;
3155 dev->driver->irq_uninstall = i915_irq_uninstall;
3156 dev->driver->irq_handler = i915_irq_handler;
20afbda2 3157 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
c2798b19 3158 } else {
a266c7d5
CW		 3159 dev->driver->irq_preinstall = i965_irq_preinstall;
3160 dev->driver->irq_postinstall = i965_irq_postinstall;
3161 dev->driver->irq_uninstall = i965_irq_uninstall;
3162 dev->driver->irq_handler = i965_irq_handler;
bac56d5b 3163 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
c2798b19 3164 }
f71d4af4
JB		 3165 dev->driver->enable_vblank = i915_enable_vblank;
3166 dev->driver->disable_vblank = i915_disable_vblank;
3167 }
3168}
20afbda2
DV		 3169
3170void intel_hpd_init(struct drm_device *dev)
3171{
3172 struct drm_i915_private *dev_priv = dev->dev_private;
821450c6
EE		 3173 struct drm_mode_config *mode_config = &dev->mode_config;
3174 struct drm_connector *connector;
b5ea2d56 3175 unsigned long irqflags;
821450c6 3176 int i;
20afbda2 3177
821450c6
EE		 3178 for (i = 1; i < HPD_NUM_PINS; i++) {
3179 dev_priv->hpd_stats[i].hpd_cnt = 0;
3180 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3181 }
3182 list_for_each_entry(connector, &mode_config->connector_list, head) {
3183 struct intel_connector *intel_connector = to_intel_connector(connector);
3184 connector->polled = intel_connector->polled;
3185 if (!connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
3186 connector->polled = DRM_CONNECTOR_POLL_HPD;
3187 }
b5ea2d56
DV		 3188
3189 /* Interrupt setup is already guaranteed to be single-threaded, this is
3190 * just to make the assert_spin_locked checks happy. */
3191 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
20afbda2
DV		 3192 if (dev_priv->display.hpd_irq_setup)
3193 dev_priv->display.hpd_irq_setup(dev);
b5ea2d56 3194 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
20afbda2 3195}
c67a470b
PZ		 3196
3197/* Disable interrupts so we can allow Package C8+. */
3198void hsw_pc8_disable_interrupts(struct drm_device *dev)
3199{
3200 struct drm_i915_private *dev_priv = dev->dev_private;
3201 unsigned long irqflags;
3202
3203 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3204
3205 dev_priv->pc8.regsave.deimr = I915_READ(DEIMR);
3206 dev_priv->pc8.regsave.sdeimr = I915_READ(SDEIMR);
3207 dev_priv->pc8.regsave.gtimr = I915_READ(GTIMR);
3208 dev_priv->pc8.regsave.gtier = I915_READ(GTIER);
3209 dev_priv->pc8.regsave.gen6_pmimr = I915_READ(GEN6_PMIMR);
3210
3211 ironlake_disable_display_irq(dev_priv, ~DE_PCH_EVENT_IVB);
3212 ibx_disable_display_interrupt(dev_priv, ~SDE_HOTPLUG_MASK_CPT);
3213 ilk_disable_gt_irq(dev_priv, 0xffffffff);
3214 snb_disable_pm_irq(dev_priv, 0xffffffff);
3215
3216 dev_priv->pc8.irqs_disabled = true;
3217
3218 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3219}
3220
3221/* Restore interrupts so we can recover from Package C8+. */
3222void hsw_pc8_restore_interrupts(struct drm_device *dev)
3223{
3224 struct drm_i915_private *dev_priv = dev->dev_private;
3225 unsigned long irqflags;
3226 uint32_t val, expected;
3227
3228 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3229
3230 val = I915_READ(DEIMR);
3231 expected = ~DE_PCH_EVENT_IVB;
3232 WARN(val != expected, "DEIMR is 0x%08x, not 0x%08x\n", val, expected);
3233
3234 val = I915_READ(SDEIMR) & ~SDE_HOTPLUG_MASK_CPT;
3235 expected = ~SDE_HOTPLUG_MASK_CPT;
3236 WARN(val != expected, "SDEIMR non-HPD bits are 0x%08x, not 0x%08x\n",
3237 val, expected);
3238
3239 val = I915_READ(GTIMR);
3240 expected = 0xffffffff;
3241 WARN(val != expected, "GTIMR is 0x%08x, not 0x%08x\n", val, expected);
3242
3243 val = I915_READ(GEN6_PMIMR);
3244 expected = 0xffffffff;
3245 WARN(val != expected, "GEN6_PMIMR is 0x%08x, not 0x%08x\n", val,
3246 expected);
3247
3248 dev_priv->pc8.irqs_disabled = false;
3249
3250 ironlake_enable_display_irq(dev_priv, ~dev_priv->pc8.regsave.deimr);
3251 ibx_enable_display_interrupt(dev_priv,
3252 ~dev_priv->pc8.regsave.sdeimr &
3253 ~SDE_HOTPLUG_MASK_CPT);
3254 ilk_enable_gt_irq(dev_priv, ~dev_priv->pc8.regsave.gtimr);
3255 snb_enable_pm_irq(dev_priv, ~dev_priv->pc8.regsave.gen6_pmimr);
3256 I915_WRITE(GTIER, dev_priv->pc8.regsave.gtier);
3257
3258 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3259}
Ubuntu Hirsute Linux kernel mirror