drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/sysrq.h>
  32 #include <linux/slab.h>
  33 #include <linux/circ_buf.h>
  34 #include <drm/drmP.h>
  35 #include <drm/i915_drm.h>
  36 #include "i915_drv.h"
  37 #include "i915_trace.h"
  38 #include "intel_drv.h"
  39
  40 /**
  41  * DOC: interrupt handling
  42  *
  43  * These functions provide the basic support for enabling and disabling the
  44  * interrupt handling support. There's a lot more functionality in i915_irq.c
  45  * and related files, but that will be described in separate chapters.
  46  */
  47
  48 static const u32 hpd_ilk[HPD_NUM_PINS] = {
  49         [HPD_PORT_A] = DE_DP_A_HOTPLUG,
  50 };
  51
  52 static const u32 hpd_ivb[HPD_NUM_PINS] = {
  53         [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
  54 };
  55
  56 static const u32 hpd_bdw[HPD_NUM_PINS] = {
  57         [HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
  58 };
  59
  60 static const u32 hpd_ibx[HPD_NUM_PINS] = {
  61         [HPD_CRT] = SDE_CRT_HOTPLUG,
  62         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
  63         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
  64         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
  65         [HPD_PORT_D] = SDE_PORTD_HOTPLUG
  66 };
  67
  68 static const u32 hpd_cpt[HPD_NUM_PINS] = {
  69         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
  70         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
  71         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  72         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  73         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
  74 };
  75
  76 static const u32 hpd_spt[HPD_NUM_PINS] = {
  77         [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
  78         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  79         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  80         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
  81         [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
  82 };
  83
  84 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
  85         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
  86         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
  87         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
  88         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
  89         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
  90         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
  91 };
  92
  93 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
  94         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  95         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
  96         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
  97         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  98         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  99         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
 100 };
 101
 102 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
 103         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
 104         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
 105         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
 106         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
 107         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
 108         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
 109 };
 110
 111 /* BXT hpd list */
 112 static const u32 hpd_bxt[HPD_NUM_PINS] = {
 113         [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
 114         [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
 115         [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
 116 };
 117
 118 /* IIR can theoretically queue up two events. Be paranoid. */
 119 #define GEN8_IRQ_RESET_NDX(type, which) do { \
 120         I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
 121         POSTING_READ(GEN8_##type##_IMR(which)); \
 122         I915_WRITE(GEN8_##type##_IER(which), 0); \
 123         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
 124         POSTING_READ(GEN8_##type##_IIR(which)); \
 125         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
 126         POSTING_READ(GEN8_##type##_IIR(which)); \
 127 } while (0)
 128
 129 #define GEN5_IRQ_RESET(type) do { \
 130         I915_WRITE(type##IMR, 0xffffffff); \
 131         POSTING_READ(type##IMR); \
 132         I915_WRITE(type##IER, 0); \
 133         I915_WRITE(type##IIR, 0xffffffff); \
 134         POSTING_READ(type##IIR); \
 135         I915_WRITE(type##IIR, 0xffffffff); \
 136         POSTING_READ(type##IIR); \
 137 } while (0)
 138
 139 /*
 140  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 141  */
 142 static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, u32 reg)
 143 {
 144         u32 val = I915_READ(reg);
 145
 146         if (val == 0)
 147                 return;
 148
 149         WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
 150              reg, val);
 151         I915_WRITE(reg, 0xffffffff);
 152         POSTING_READ(reg);
 153         I915_WRITE(reg, 0xffffffff);
 154         POSTING_READ(reg);
 155 }
 156
 157 #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
 158         gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
 159         I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
 160         I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
 161         POSTING_READ(GEN8_##type##_IMR(which)); \
 162 } while (0)
 163
 164 #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
 165         gen5_assert_iir_is_zero(dev_priv, type##IIR); \
 166         I915_WRITE(type##IER, (ier_val)); \
 167         I915_WRITE(type##IMR, (imr_val)); \
 168         POSTING_READ(type##IMR); \
 169 } while (0)
 170
 171 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
 172
 173 /* For display hotplug interrupt */
 174 static inline void
 175 i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
 176                                      uint32_t mask,
 177                                      uint32_t bits)
 178 {
 179         uint32_t val;
 180
 181         assert_spin_locked(&dev_priv->irq_lock);
 182         WARN_ON(bits & ~mask);
 183
 184         val = I915_READ(PORT_HOTPLUG_EN);
 185         val &= ~mask;
 186         val |= bits;
 187         I915_WRITE(PORT_HOTPLUG_EN, val);
 188 }
 189
 190 /**
 191  * i915_hotplug_interrupt_update - update hotplug interrupt enable
 192  * @dev_priv: driver private
 193  * @mask: bits to update
 194  * @bits: bits to enable
 195  * NOTE: the HPD enable bits are modified both inside and outside
 196  * of an interrupt context. To avoid that read-modify-write cycles
 197  * interfer, these bits are protected by a spinlock. Since this
 198  * function is usually not called from a context where the lock is
 199  * held already, this function acquires the lock itself. A non-locking
 200  * version is also available.
 201  */
 202 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
 203                                    uint32_t mask,
 204                                    uint32_t bits)
 205 {
 206         spin_lock_irq(&dev_priv->irq_lock);
 207         i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
 208         spin_unlock_irq(&dev_priv->irq_lock);
 209 }
 210
 211 /**
 212  * ilk_update_display_irq - update DEIMR
 213  * @dev_priv: driver private
 214  * @interrupt_mask: mask of interrupt bits to update
 215  * @enabled_irq_mask: mask of interrupt bits to enable
 216  */
 217 static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
 218                                    uint32_t interrupt_mask,
 219                                    uint32_t enabled_irq_mask)
 220 {
 221         uint32_t new_val;
 222
 223         assert_spin_locked(&dev_priv->irq_lock);
 224
 225         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 226
 227         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 228                 return;
 229
 230         new_val = dev_priv->irq_mask;
 231         new_val &= ~interrupt_mask;
 232         new_val |= (~enabled_irq_mask & interrupt_mask);
 233
 234         if (new_val != dev_priv->irq_mask) {
 235                 dev_priv->irq_mask = new_val;
 236                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 237                 POSTING_READ(DEIMR);
 238         }
 239 }
 240
 241 void
 242 ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 243 {
 244         ilk_update_display_irq(dev_priv, mask, mask);
 245 }
 246
 247 void
 248 ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 249 {
 250         ilk_update_display_irq(dev_priv, mask, 0);
 251 }
 252
 253 /**
 254  * ilk_update_gt_irq - update GTIMR
 255  * @dev_priv: driver private
 256  * @interrupt_mask: mask of interrupt bits to update
 257  * @enabled_irq_mask: mask of interrupt bits to enable
 258  */
 259 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
 260                               uint32_t interrupt_mask,
 261                               uint32_t enabled_irq_mask)
 262 {
 263         assert_spin_locked(&dev_priv->irq_lock);
 264
 265         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 266
 267         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 268                 return;
 269
 270         dev_priv->gt_irq_mask &= ~interrupt_mask;
 271         dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
 272         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 273         POSTING_READ(GTIMR);
 274 }
 275
 276 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 277 {
 278         ilk_update_gt_irq(dev_priv, mask, mask);
 279 }
 280
 281 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 282 {
 283         ilk_update_gt_irq(dev_priv, mask, 0);
 284 }
 285
 286 static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
 287 {
 288         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
 289 }
 290
 291 static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
 292 {
 293         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
 294 }
 295
 296 static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
 297 {
 298         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
 299 }
 300
 301 /**
 302   * snb_update_pm_irq - update GEN6_PMIMR
 303   * @dev_priv: driver private
 304   * @interrupt_mask: mask of interrupt bits to update
 305   * @enabled_irq_mask: mask of interrupt bits to enable
 306   */
 307 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 308                               uint32_t interrupt_mask,
 309                               uint32_t enabled_irq_mask)
 310 {
 311         uint32_t new_val;
 312
 313         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 314
 315         assert_spin_locked(&dev_priv->irq_lock);
 316
 317         new_val = dev_priv->pm_irq_mask;
 318         new_val &= ~interrupt_mask;
 319         new_val |= (~enabled_irq_mask & interrupt_mask);
 320
 321         if (new_val != dev_priv->pm_irq_mask) {
 322                 dev_priv->pm_irq_mask = new_val;
 323                 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
 324                 POSTING_READ(gen6_pm_imr(dev_priv));
 325         }
 326 }
 327
 328 void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 329 {
 330         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 331                 return;
 332
 333         snb_update_pm_irq(dev_priv, mask, mask);
 334 }
 335
 336 static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
 337                                   uint32_t mask)
 338 {
 339         snb_update_pm_irq(dev_priv, mask, 0);
 340 }
 341
 342 void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 343 {
 344         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 345                 return;
 346
 347         __gen6_disable_pm_irq(dev_priv, mask);
 348 }
 349
 350 void gen6_reset_rps_interrupts(struct drm_device *dev)
 351 {
 352         struct drm_i915_private *dev_priv = dev->dev_private;
 353         uint32_t reg = gen6_pm_iir(dev_priv);
 354
 355         spin_lock_irq(&dev_priv->irq_lock);
 356         I915_WRITE(reg, dev_priv->pm_rps_events);
 357         I915_WRITE(reg, dev_priv->pm_rps_events);
 358         POSTING_READ(reg);
 359         dev_priv->rps.pm_iir = 0;
 360         spin_unlock_irq(&dev_priv->irq_lock);
 361 }
 362
 363 void gen6_enable_rps_interrupts(struct drm_device *dev)
 364 {
 365         struct drm_i915_private *dev_priv = dev->dev_private;
 366
 367         spin_lock_irq(&dev_priv->irq_lock);
 368
 369         WARN_ON(dev_priv->rps.pm_iir);
 370         WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
 371         dev_priv->rps.interrupts_enabled = true;
 372         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
 373                                 dev_priv->pm_rps_events);
 374         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 375
 376         spin_unlock_irq(&dev_priv->irq_lock);
 377 }
 378
 379 u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
 380 {
 381         /*
 382          * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
 383          * if GEN6_PM_UP_EI_EXPIRED is masked.
 384          *
 385          * TODO: verify if this can be reproduced on VLV,CHV.
 386          */
 387         if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
 388                 mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
 389
 390         if (INTEL_INFO(dev_priv)->gen >= 8)
 391                 mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
 392
 393         return mask;
 394 }
 395
 396 void gen6_disable_rps_interrupts(struct drm_device *dev)
 397 {
 398         struct drm_i915_private *dev_priv = dev->dev_private;
 399
 400         spin_lock_irq(&dev_priv->irq_lock);
 401         dev_priv->rps.interrupts_enabled = false;
 402         spin_unlock_irq(&dev_priv->irq_lock);
 403
 404         cancel_work_sync(&dev_priv->rps.work);
 405
 406         spin_lock_irq(&dev_priv->irq_lock);
 407
 408         I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
 409
 410         __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 411         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
 412                                 ~dev_priv->pm_rps_events);
 413
 414         spin_unlock_irq(&dev_priv->irq_lock);
 415
 416         synchronize_irq(dev->irq);
 417 }
 418
 419 /**
 420   * bdw_update_port_irq - update DE port interrupt
 421   * @dev_priv: driver private
 422   * @interrupt_mask: mask of interrupt bits to update
 423   * @enabled_irq_mask: mask of interrupt bits to enable
 424   */
 425 static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
 426                                 uint32_t interrupt_mask,
 427                                 uint32_t enabled_irq_mask)
 428 {
 429         uint32_t new_val;
 430         uint32_t old_val;
 431
 432         assert_spin_locked(&dev_priv->irq_lock);
 433
 434         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 435
 436         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 437                 return;
 438
 439         old_val = I915_READ(GEN8_DE_PORT_IMR);
 440
 441         new_val = old_val;
 442         new_val &= ~interrupt_mask;
 443         new_val |= (~enabled_irq_mask & interrupt_mask);
 444
 445         if (new_val != old_val) {
 446                 I915_WRITE(GEN8_DE_PORT_IMR, new_val);
 447                 POSTING_READ(GEN8_DE_PORT_IMR);
 448         }
 449 }
 450
 451 /**
 452  * ibx_display_interrupt_update - update SDEIMR
 453  * @dev_priv: driver private
 454  * @interrupt_mask: mask of interrupt bits to update
 455  * @enabled_irq_mask: mask of interrupt bits to enable
 456  */
 457 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 458                                   uint32_t interrupt_mask,
 459                                   uint32_t enabled_irq_mask)
 460 {
 461         uint32_t sdeimr = I915_READ(SDEIMR);
 462         sdeimr &= ~interrupt_mask;
 463         sdeimr |= (~enabled_irq_mask & interrupt_mask);
 464
 465         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 466
 467         assert_spin_locked(&dev_priv->irq_lock);
 468
 469         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 470                 return;
 471
 472         I915_WRITE(SDEIMR, sdeimr);
 473         POSTING_READ(SDEIMR);
 474 }
 475
 476 static void
 477 __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 478                        u32 enable_mask, u32 status_mask)
 479 {
 480         u32 reg = PIPESTAT(pipe);
 481         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 482
 483         assert_spin_locked(&dev_priv->irq_lock);
 484         WARN_ON(!intel_irqs_enabled(dev_priv));
 485
 486         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 487                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 488                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 489                       pipe_name(pipe), enable_mask, status_mask))
 490                 return;
 491
 492         if ((pipestat & enable_mask) == enable_mask)
 493                 return;
 494
 495         dev_priv->pipestat_irq_mask[pipe] |= status_mask;
 496
 497         /* Enable the interrupt, clear any pending status */
 498         pipestat |= enable_mask | status_mask;
 499         I915_WRITE(reg, pipestat);
 500         POSTING_READ(reg);
 501 }
 502
 503 static void
 504 __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 505                         u32 enable_mask, u32 status_mask)
 506 {
 507         u32 reg = PIPESTAT(pipe);
 508         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 509
 510         assert_spin_locked(&dev_priv->irq_lock);
 511         WARN_ON(!intel_irqs_enabled(dev_priv));
 512
 513         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 514                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 515                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 516                       pipe_name(pipe), enable_mask, status_mask))
 517                 return;
 518
 519         if ((pipestat & enable_mask) == 0)
 520                 return;
 521
 522         dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
 523
 524         pipestat &= ~enable_mask;
 525         I915_WRITE(reg, pipestat);
 526         POSTING_READ(reg);
 527 }
 528
 529 static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
 530 {
 531         u32 enable_mask = status_mask << 16;
 532
 533         /*
 534          * On pipe A we don't support the PSR interrupt yet,
 535          * on pipe B and C the same bit MBZ.
 536          */
 537         if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
 538                 return 0;
 539         /*
 540          * On pipe B and C we don't support the PSR interrupt yet, on pipe
 541          * A the same bit is for perf counters which we don't use either.
 542          */
 543         if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
 544                 return 0;
 545
 546         enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
 547                          SPRITE0_FLIP_DONE_INT_EN_VLV |
 548                          SPRITE1_FLIP_DONE_INT_EN_VLV);
 549         if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
 550                 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
 551         if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
 552                 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
 553
 554         return enable_mask;
 555 }
 556
 557 void
 558 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 559                      u32 status_mask)
 560 {
 561         u32 enable_mask;
 562
 563         if (IS_VALLEYVIEW(dev_priv->dev))
 564                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 565                                                            status_mask);
 566         else
 567                 enable_mask = status_mask << 16;
 568         __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
 569 }
 570
 571 void
 572 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 573                       u32 status_mask)
 574 {
 575         u32 enable_mask;
 576
 577         if (IS_VALLEYVIEW(dev_priv->dev))
 578                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 579                                                            status_mask);
 580         else
 581                 enable_mask = status_mask << 16;
 582         __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
 583 }
 584
 585 /**
 586  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 587  * @dev: drm device
 588  */
 589 static void i915_enable_asle_pipestat(struct drm_device *dev)
 590 {
 591         struct drm_i915_private *dev_priv = dev->dev_private;
 592
 593         if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
 594                 return;
 595
 596         spin_lock_irq(&dev_priv->irq_lock);
 597
 598         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
 599         if (INTEL_INFO(dev)->gen >= 4)
 600                 i915_enable_pipestat(dev_priv, PIPE_A,
 601                                      PIPE_LEGACY_BLC_EVENT_STATUS);
 602
 603         spin_unlock_irq(&dev_priv->irq_lock);
 604 }
 605
 606 /*
 607  * This timing diagram depicts the video signal in and
 608  * around the vertical blanking period.
 609  *
 610  * Assumptions about the fictitious mode used in this example:
 611  *  vblank_start >= 3
 612  *  vsync_start = vblank_start + 1
 613  *  vsync_end = vblank_start + 2
 614  *  vtotal = vblank_start + 3
 615  *
 616  *           start of vblank:
 617  *           latch double buffered registers
 618  *           increment frame counter (ctg+)
 619  *           generate start of vblank interrupt (gen4+)
 620  *           |
 621  *           |          frame start:
 622  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 623  *           |          may be shifted forward 1-3 extra lines via PIPECONF
 624  *           |          |
 625  *           |          |  start of vsync:
 626  *           |          |  generate vsync interrupt
 627  *           |          |  |
 628  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 629  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 630  * ----va---> <-----------------vb--------------------> <--------va-------------
 631  *       |          |       <----vs----->                     |
 632  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 633  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 634  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 635  *       |          |                                         |
 636  *       last visible pixel                                   first visible pixel
 637  *                  |                                         increment frame counter (gen3/4)
 638  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 639  *
 640  * x  = horizontal active
 641  * _  = horizontal blanking
 642  * hs = horizontal sync
 643  * va = vertical active
 644  * vb = vertical blanking
 645  * vs = vertical sync
 646  * vbs = vblank_start (number)
 647  *
 648  * Summary:
 649  * - most events happen at the start of horizontal sync
 650  * - frame start happens at the start of horizontal blank, 1-4 lines
 651  *   (depending on PIPECONF settings) after the start of vblank
 652  * - gen3/4 pixel and frame counter are synchronized with the start
 653  *   of horizontal active on the first line of vertical active
 654  */
 655
 656 static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
 657 {
 658         /* Gen2 doesn't have a hardware frame counter */
 659         return 0;
 660 }
 661
 662 /* Called from drm generic code, passed a 'crtc', which
 663  * we use as a pipe index
 664  */
 665 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 666 {
 667         struct drm_i915_private *dev_priv = dev->dev_private;
 668         unsigned long high_frame;
 669         unsigned long low_frame;
 670         u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
 671         struct intel_crtc *intel_crtc =
 672                 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 673         const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
 674
 675         htotal = mode->crtc_htotal;
 676         hsync_start = mode->crtc_hsync_start;
 677         vbl_start = mode->crtc_vblank_start;
 678         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 679                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 680
 681         /* Convert to pixel count */
 682         vbl_start *= htotal;
 683
 684         /* Start of vblank event occurs at start of hsync */
 685         vbl_start -= htotal - hsync_start;
 686
 687         high_frame = PIPEFRAME(pipe);
 688         low_frame = PIPEFRAMEPIXEL(pipe);
 689
 690         /*
 691          * High & low register fields aren't synchronized, so make sure
 692          * we get a low value that's stable across two reads of the high
 693          * register.
 694          */
 695         do {
 696                 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 697                 low   = I915_READ(low_frame);
 698                 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 699         } while (high1 != high2);
 700
 701         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 702         pixel = low & PIPE_PIXEL_MASK;
 703         low >>= PIPE_FRAME_LOW_SHIFT;
 704
 705         /*
 706          * The frame counter increments at beginning of active.
 707          * Cook up a vblank counter by also checking the pixel
 708          * counter against vblank start.
 709          */
 710         return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
 711 }
 712
 713 static u32 g4x_get_vblank_counter(struct drm_device *dev, int pipe)
 714 {
 715         struct drm_i915_private *dev_priv = dev->dev_private;
 716         int reg = PIPE_FRMCOUNT_G4X(pipe);
 717
 718         return I915_READ(reg);
 719 }
 720
 721 /* raw reads, only for fast reads of display block, no need for forcewake etc. */
 722 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
 723
 724 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 725 {
 726         struct drm_device *dev = crtc->base.dev;
 727         struct drm_i915_private *dev_priv = dev->dev_private;
 728         const struct drm_display_mode *mode = &crtc->base.hwmode;
 729         enum pipe pipe = crtc->pipe;
 730         int position, vtotal;
 731
 732         vtotal = mode->crtc_vtotal;
 733         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 734                 vtotal /= 2;
 735
 736         if (IS_GEN2(dev))
 737                 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
 738         else
 739                 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
 740
 741         /*
 742          * On HSW, the DSL reg (0x70000) appears to return 0 if we
 743          * read it just before the start of vblank.  So try it again
 744          * so we don't accidentally end up spanning a vblank frame
 745          * increment, causing the pipe_update_end() code to squak at us.
 746          *
 747          * The nature of this problem means we can't simply check the ISR
 748          * bit and return the vblank start value; nor can we use the scanline
 749          * debug register in the transcoder as it appears to have the same
 750          * problem.  We may need to extend this to include other platforms,
 751          * but so far testing only shows the problem on HSW.
 752          */
 753         if (IS_HASWELL(dev) && !position) {
 754                 int i, temp;
 755
 756                 for (i = 0; i < 100; i++) {
 757                         udelay(1);
 758                         temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
 759                                 DSL_LINEMASK_GEN3;
 760                         if (temp != position) {
 761                                 position = temp;
 762                                 break;
 763                         }
 764                 }
 765         }
 766
 767         /*
 768          * See update_scanline_offset() for the details on the
 769          * scanline_offset adjustment.
 770          */
 771         return (position + crtc->scanline_offset) % vtotal;
 772 }
 773
 774 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 775                                     unsigned int flags, int *vpos, int *hpos,
 776                                     ktime_t *stime, ktime_t *etime,
 777                                     const struct drm_display_mode *mode)
 778 {
 779         struct drm_i915_private *dev_priv = dev->dev_private;
 780         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 781         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 782         int position;
 783         int vbl_start, vbl_end, hsync_start, htotal, vtotal;
 784         bool in_vbl = true;
 785         int ret = 0;
 786         unsigned long irqflags;
 787
 788         if (WARN_ON(!mode->crtc_clock)) {
 789                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 790                                  "pipe %c\n", pipe_name(pipe));
 791                 return 0;
 792         }
 793
 794         htotal = mode->crtc_htotal;
 795         hsync_start = mode->crtc_hsync_start;
 796         vtotal = mode->crtc_vtotal;
 797         vbl_start = mode->crtc_vblank_start;
 798         vbl_end = mode->crtc_vblank_end;
 799
 800         if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
 801                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 802                 vbl_end /= 2;
 803                 vtotal /= 2;
 804         }
 805
 806         ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 807
 808         /*
 809          * Lock uncore.lock, as we will do multiple timing critical raw
 810          * register reads, potentially with preemption disabled, so the
 811          * following code must not block on uncore.lock.
 812          */
 813         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 814
 815         /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
 816
 817         /* Get optional system timestamp before query. */
 818         if (stime)
 819                 *stime = ktime_get();
 820
 821         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 822                 /* No obvious pixelcount register. Only query vertical
 823                  * scanout position from Display scan line register.
 824                  */
 825                 position = __intel_get_crtc_scanline(intel_crtc);
 826         } else {
 827                 /* Have access to pixelcount since start of frame.
 828                  * We can split this into vertical and horizontal
 829                  * scanout position.
 830                  */
 831                 position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 832
 833                 /* convert to pixel counts */
 834                 vbl_start *= htotal;
 835                 vbl_end *= htotal;
 836                 vtotal *= htotal;
 837
 838                 /*
 839                  * In interlaced modes, the pixel counter counts all pixels,
 840                  * so one field will have htotal more pixels. In order to avoid
 841                  * the reported position from jumping backwards when the pixel
 842                  * counter is beyond the length of the shorter field, just
 843                  * clamp the position the length of the shorter field. This
 844                  * matches how the scanline counter based position works since
 845                  * the scanline counter doesn't count the two half lines.
 846                  */
 847                 if (position >= vtotal)
 848                         position = vtotal - 1;
 849
 850                 /*
 851                  * Start of vblank interrupt is triggered at start of hsync,
 852                  * just prior to the first active line of vblank. However we
 853                  * consider lines to start at the leading edge of horizontal
 854                  * active. So, should we get here before we've crossed into
 855                  * the horizontal active of the first line in vblank, we would
 856                  * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
 857                  * always add htotal-hsync_start to the current pixel position.
 858                  */
 859                 position = (position + htotal - hsync_start) % vtotal;
 860         }
 861
 862         /* Get optional system timestamp after query. */
 863         if (etime)
 864                 *etime = ktime_get();
 865
 866         /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
 867
 868         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 869
 870         in_vbl = position >= vbl_start && position < vbl_end;
 871
 872         /*
 873          * While in vblank, position will be negative
 874          * counting up towards 0 at vbl_end. And outside
 875          * vblank, position will be positive counting
 876          * up since vbl_end.
 877          */
 878         if (position >= vbl_start)
 879                 position -= vbl_end;
 880         else
 881                 position += vtotal - vbl_end;
 882
 883         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 884                 *vpos = position;
 885                 *hpos = 0;
 886         } else {
 887                 *vpos = position / htotal;
 888                 *hpos = position - (*vpos * htotal);
 889         }
 890
 891         /* In vblank? */
 892         if (in_vbl)
 893                 ret |= DRM_SCANOUTPOS_IN_VBLANK;
 894
 895         return ret;
 896 }
 897
 898 int intel_get_crtc_scanline(struct intel_crtc *crtc)
 899 {
 900         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 901         unsigned long irqflags;
 902         int position;
 903
 904         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 905         position = __intel_get_crtc_scanline(crtc);
 906         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 907
 908         return position;
 909 }
 910
 911 static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
 912                               int *max_error,
 913                               struct timeval *vblank_time,
 914                               unsigned flags)
 915 {
 916         struct drm_crtc *crtc;
 917
 918         if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
 919                 DRM_ERROR("Invalid crtc %d\n", pipe);
 920                 return -EINVAL;
 921         }
 922
 923         /* Get drm_crtc to timestamp: */
 924         crtc = intel_get_crtc_for_pipe(dev, pipe);
 925         if (crtc == NULL) {
 926                 DRM_ERROR("Invalid crtc %d\n", pipe);
 927                 return -EINVAL;
 928         }
 929
 930         if (!crtc->hwmode.crtc_clock) {
 931                 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
 932                 return -EBUSY;
 933         }
 934
 935         /* Helper routine in DRM core does all the work: */
 936         return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 937                                                      vblank_time, flags,
 938                                                      &crtc->hwmode);
 939 }
 940
 941 static void ironlake_rps_change_irq_handler(struct drm_device *dev)
 942 {
 943         struct drm_i915_private *dev_priv = dev->dev_private;
 944         u32 busy_up, busy_down, max_avg, min_avg;
 945         u8 new_delay;
 946
 947         spin_lock(&mchdev_lock);
 948
 949         I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
 950
 951         new_delay = dev_priv->ips.cur_delay;
 952
 953         I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 954         busy_up = I915_READ(RCPREVBSYTUPAVG);
 955         busy_down = I915_READ(RCPREVBSYTDNAVG);
 956         max_avg = I915_READ(RCBMAXAVG);
 957         min_avg = I915_READ(RCBMINAVG);
 958
 959         /* Handle RCS change request from hw */
 960         if (busy_up > max_avg) {
 961                 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
 962                         new_delay = dev_priv->ips.cur_delay - 1;
 963                 if (new_delay < dev_priv->ips.max_delay)
 964                         new_delay = dev_priv->ips.max_delay;
 965         } else if (busy_down < min_avg) {
 966                 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
 967                         new_delay = dev_priv->ips.cur_delay + 1;
 968                 if (new_delay > dev_priv->ips.min_delay)
 969                         new_delay = dev_priv->ips.min_delay;
 970         }
 971
 972         if (ironlake_set_drps(dev, new_delay))
 973                 dev_priv->ips.cur_delay = new_delay;
 974
 975         spin_unlock(&mchdev_lock);
 976
 977         return;
 978 }
 979
 980 static void notify_ring(struct intel_engine_cs *ring)
 981 {
 982         if (!intel_ring_initialized(ring))
 983                 return;
 984
 985         trace_i915_gem_request_notify(ring);
 986
 987         wake_up_all(&ring->irq_queue);
 988 }
 989
 990 static void vlv_c0_read(struct drm_i915_private *dev_priv,
 991                         struct intel_rps_ei *ei)
 992 {
 993         ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
 994         ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
 995         ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
 996 }
 997
 998 static bool vlv_c0_above(struct drm_i915_private *dev_priv,
 999                          const struct intel_rps_ei *old,
1000                          const struct intel_rps_ei *now,
1001                          int threshold)
1002 {
1003         u64 time, c0;
1004         unsigned int mul = 100;
1005
1006         if (old->cz_clock == 0)
1007                 return false;
1008
1009         if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
1010                 mul <<= 8;
1011
1012         time = now->cz_clock - old->cz_clock;
1013         time *= threshold * dev_priv->czclk_freq;
1014
1015         /* Workload can be split between render + media, e.g. SwapBuffers
1016          * being blitted in X after being rendered in mesa. To account for
1017          * this we need to combine both engines into our activity counter.
1018          */
1019         c0 = now->render_c0 - old->render_c0;
1020         c0 += now->media_c0 - old->media_c0;
1021         c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC;
1022
1023         return c0 >= time;
1024 }
1025
1026 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1027 {
1028         vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
1029         dev_priv->rps.up_ei = dev_priv->rps.down_ei;
1030 }
1031
1032 static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
1033 {
1034         struct intel_rps_ei now;
1035         u32 events = 0;
1036
1037         if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
1038                 return 0;
1039
1040         vlv_c0_read(dev_priv, &now);
1041         if (now.cz_clock == 0)
1042                 return 0;
1043
1044         if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
1045                 if (!vlv_c0_above(dev_priv,
1046                                   &dev_priv->rps.down_ei, &now,
1047                                   dev_priv->rps.down_threshold))
1048                         events |= GEN6_PM_RP_DOWN_THRESHOLD;
1049                 dev_priv->rps.down_ei = now;
1050         }
1051
1052         if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
1053                 if (vlv_c0_above(dev_priv,
1054                                  &dev_priv->rps.up_ei, &now,
1055                                  dev_priv->rps.up_threshold))
1056                         events |= GEN6_PM_RP_UP_THRESHOLD;
1057                 dev_priv->rps.up_ei = now;
1058         }
1059
1060         return events;
1061 }
1062
1063 static bool any_waiters(struct drm_i915_private *dev_priv)
1064 {
1065         struct intel_engine_cs *ring;
1066         int i;
1067
1068         for_each_ring(ring, dev_priv, i)
1069                 if (ring->irq_refcount)
1070                         return true;
1071
1072         return false;
1073 }
1074
1075 static void gen6_pm_rps_work(struct work_struct *work)
1076 {
1077         struct drm_i915_private *dev_priv =
1078                 container_of(work, struct drm_i915_private, rps.work);
1079         bool client_boost;
1080         int new_delay, adj, min, max;
1081         u32 pm_iir;
1082
1083         spin_lock_irq(&dev_priv->irq_lock);
1084         /* Speed up work cancelation during disabling rps interrupts. */
1085         if (!dev_priv->rps.interrupts_enabled) {
1086                 spin_unlock_irq(&dev_priv->irq_lock);
1087                 return;
1088         }
1089         pm_iir = dev_priv->rps.pm_iir;
1090         dev_priv->rps.pm_iir = 0;
1091         /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
1092         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
1093         client_boost = dev_priv->rps.client_boost;
1094         dev_priv->rps.client_boost = false;
1095         spin_unlock_irq(&dev_priv->irq_lock);
1096
1097         /* Make sure we didn't queue anything we're not going to process. */
1098         WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1099
1100         if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1101                 return;
1102
1103         mutex_lock(&dev_priv->rps.hw_lock);
1104
1105         pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
1106
1107         adj = dev_priv->rps.last_adj;
1108         new_delay = dev_priv->rps.cur_freq;
1109         min = dev_priv->rps.min_freq_softlimit;
1110         max = dev_priv->rps.max_freq_softlimit;
1111
1112         if (client_boost) {
1113                 new_delay = dev_priv->rps.max_freq_softlimit;
1114                 adj = 0;
1115         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1116                 if (adj > 0)
1117                         adj *= 2;
1118                 else /* CHV needs even encode values */
1119                         adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1120                 /*
1121                  * For better performance, jump directly
1122                  * to RPe if we're below it.
1123                  */
1124                 if (new_delay < dev_priv->rps.efficient_freq - adj) {
1125                         new_delay = dev_priv->rps.efficient_freq;
1126                         adj = 0;
1127                 }
1128         } else if (any_waiters(dev_priv)) {
1129                 adj = 0;
1130         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1131                 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
1132                         new_delay = dev_priv->rps.efficient_freq;
1133                 else
1134                         new_delay = dev_priv->rps.min_freq_softlimit;
1135                 adj = 0;
1136         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1137                 if (adj < 0)
1138                         adj *= 2;
1139                 else /* CHV needs even encode values */
1140                         adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1141         } else { /* unknown event */
1142                 adj = 0;
1143         }
1144
1145         dev_priv->rps.last_adj = adj;
1146
1147         /* sysfs frequency interfaces may have snuck in while servicing the
1148          * interrupt
1149          */
1150         new_delay += adj;
1151         new_delay = clamp_t(int, new_delay, min, max);
1152
1153         intel_set_rps(dev_priv->dev, new_delay);
1154
1155         mutex_unlock(&dev_priv->rps.hw_lock);
1156 }
1157
1158
1159 /**
1160  * ivybridge_parity_work - Workqueue called when a parity error interrupt
1161  * occurred.
1162  * @work: workqueue struct
1163  *
1164  * Doesn't actually do anything except notify userspace. As a consequence of
1165  * this event, userspace should try to remap the bad rows since statistically
1166  * it is likely the same row is more likely to go bad again.
1167  */
1168 static void ivybridge_parity_work(struct work_struct *work)
1169 {
1170         struct drm_i915_private *dev_priv =
1171                 container_of(work, struct drm_i915_private, l3_parity.error_work);
1172         u32 error_status, row, bank, subbank;
1173         char *parity_event[6];
1174         uint32_t misccpctl;
1175         uint8_t slice = 0;
1176
1177         /* We must turn off DOP level clock gating to access the L3 registers.
1178          * In order to prevent a get/put style interface, acquire struct mutex
1179          * any time we access those registers.
1180          */
1181         mutex_lock(&dev_priv->dev->struct_mutex);
1182
1183         /* If we've screwed up tracking, just let the interrupt fire again */
1184         if (WARN_ON(!dev_priv->l3_parity.which_slice))
1185                 goto out;
1186
1187         misccpctl = I915_READ(GEN7_MISCCPCTL);
1188         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1189         POSTING_READ(GEN7_MISCCPCTL);
1190
1191         while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1192                 u32 reg;
1193
1194                 slice--;
1195                 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
1196                         break;
1197
1198                 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1199
1200                 reg = GEN7_L3CDERRST1 + (slice * 0x200);
1201
1202                 error_status = I915_READ(reg);
1203                 row = GEN7_PARITY_ERROR_ROW(error_status);
1204                 bank = GEN7_PARITY_ERROR_BANK(error_status);
1205                 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1206
1207                 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1208                 POSTING_READ(reg);
1209
1210                 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1211                 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1212                 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1213                 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1214                 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1215                 parity_event[5] = NULL;
1216
1217                 kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1218                                    KOBJ_CHANGE, parity_event);
1219
1220                 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1221                           slice, row, bank, subbank);
1222
1223                 kfree(parity_event[4]);
1224                 kfree(parity_event[3]);
1225                 kfree(parity_event[2]);
1226                 kfree(parity_event[1]);
1227         }
1228
1229         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1230
1231 out:
1232         WARN_ON(dev_priv->l3_parity.which_slice);
1233         spin_lock_irq(&dev_priv->irq_lock);
1234         gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1235         spin_unlock_irq(&dev_priv->irq_lock);
1236
1237         mutex_unlock(&dev_priv->dev->struct_mutex);
1238 }
1239
1240 static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1241 {
1242         struct drm_i915_private *dev_priv = dev->dev_private;
1243
1244         if (!HAS_L3_DPF(dev))
1245                 return;
1246
1247         spin_lock(&dev_priv->irq_lock);
1248         gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1249         spin_unlock(&dev_priv->irq_lock);
1250
1251         iir &= GT_PARITY_ERROR(dev);
1252         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1253                 dev_priv->l3_parity.which_slice |= 1 << 1;
1254
1255         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1256                 dev_priv->l3_parity.which_slice |= 1 << 0;
1257
1258         queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1259 }
1260
1261 static void ilk_gt_irq_handler(struct drm_device *dev,
1262                                struct drm_i915_private *dev_priv,
1263                                u32 gt_iir)
1264 {
1265         if (gt_iir &
1266             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1267                 notify_ring(&dev_priv->ring[RCS]);
1268         if (gt_iir & ILK_BSD_USER_INTERRUPT)
1269                 notify_ring(&dev_priv->ring[VCS]);
1270 }
1271
1272 static void snb_gt_irq_handler(struct drm_device *dev,
1273                                struct drm_i915_private *dev_priv,
1274                                u32 gt_iir)
1275 {
1276
1277         if (gt_iir &
1278             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1279                 notify_ring(&dev_priv->ring[RCS]);
1280         if (gt_iir & GT_BSD_USER_INTERRUPT)
1281                 notify_ring(&dev_priv->ring[VCS]);
1282         if (gt_iir & GT_BLT_USER_INTERRUPT)
1283                 notify_ring(&dev_priv->ring[BCS]);
1284
1285         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1286                       GT_BSD_CS_ERROR_INTERRUPT |
1287                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
1288                 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1289
1290         if (gt_iir & GT_PARITY_ERROR(dev))
1291                 ivybridge_parity_error_irq_handler(dev, gt_iir);
1292 }
1293
1294 static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1295                                        u32 master_ctl)
1296 {
1297         irqreturn_t ret = IRQ_NONE;
1298
1299         if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1300                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
1301                 if (tmp) {
1302                         I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
1303                         ret = IRQ_HANDLED;
1304
1305                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1306                                 intel_lrc_irq_handler(&dev_priv->ring[RCS]);
1307                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1308                                 notify_ring(&dev_priv->ring[RCS]);
1309
1310                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1311                                 intel_lrc_irq_handler(&dev_priv->ring[BCS]);
1312                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1313                                 notify_ring(&dev_priv->ring[BCS]);
1314                 } else
1315                         DRM_ERROR("The master control interrupt lied (GT0)!\n");
1316         }
1317
1318         if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1319                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
1320                 if (tmp) {
1321                         I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
1322                         ret = IRQ_HANDLED;
1323
1324                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1325                                 intel_lrc_irq_handler(&dev_priv->ring[VCS]);
1326                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1327                                 notify_ring(&dev_priv->ring[VCS]);
1328
1329                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1330                                 intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
1331                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1332                                 notify_ring(&dev_priv->ring[VCS2]);
1333                 } else
1334                         DRM_ERROR("The master control interrupt lied (GT1)!\n");
1335         }
1336
1337         if (master_ctl & GEN8_GT_VECS_IRQ) {
1338                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
1339                 if (tmp) {
1340                         I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
1341                         ret = IRQ_HANDLED;
1342
1343                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1344                                 intel_lrc_irq_handler(&dev_priv->ring[VECS]);
1345                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1346                                 notify_ring(&dev_priv->ring[VECS]);
1347                 } else
1348                         DRM_ERROR("The master control interrupt lied (GT3)!\n");
1349         }
1350
1351         if (master_ctl & GEN8_GT_PM_IRQ) {
1352                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
1353                 if (tmp & dev_priv->pm_rps_events) {
1354                         I915_WRITE_FW(GEN8_GT_IIR(2),
1355                                       tmp & dev_priv->pm_rps_events);
1356                         ret = IRQ_HANDLED;
1357                         gen6_rps_irq_handler(dev_priv, tmp);
1358                 } else
1359                         DRM_ERROR("The master control interrupt lied (PM)!\n");
1360         }
1361
1362         return ret;
1363 }
1364
1365 static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
1366 {
1367         switch (port) {
1368         case PORT_A:
1369                 return val & PORTA_HOTPLUG_LONG_DETECT;
1370         case PORT_B:
1371                 return val & PORTB_HOTPLUG_LONG_DETECT;
1372         case PORT_C:
1373                 return val & PORTC_HOTPLUG_LONG_DETECT;
1374         default:
1375                 return false;
1376         }
1377 }
1378
1379 static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
1380 {
1381         switch (port) {
1382         case PORT_E:
1383                 return val & PORTE_HOTPLUG_LONG_DETECT;
1384         default:
1385                 return false;
1386         }
1387 }
1388
1389 static bool spt_port_hotplug_long_detect(enum port port, u32 val)
1390 {
1391         switch (port) {
1392         case PORT_A:
1393                 return val & PORTA_HOTPLUG_LONG_DETECT;
1394         case PORT_B:
1395                 return val & PORTB_HOTPLUG_LONG_DETECT;
1396         case PORT_C:
1397                 return val & PORTC_HOTPLUG_LONG_DETECT;
1398         case PORT_D:
1399                 return val & PORTD_HOTPLUG_LONG_DETECT;
1400         default:
1401                 return false;
1402         }
1403 }
1404
1405 static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
1406 {
1407         switch (port) {
1408         case PORT_A:
1409                 return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1410         default:
1411                 return false;
1412         }
1413 }
1414
1415 static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1416 {
1417         switch (port) {
1418         case PORT_B:
1419                 return val & PORTB_HOTPLUG_LONG_DETECT;
1420         case PORT_C:
1421                 return val & PORTC_HOTPLUG_LONG_DETECT;
1422         case PORT_D:
1423                 return val & PORTD_HOTPLUG_LONG_DETECT;
1424         default:
1425                 return false;
1426         }
1427 }
1428
1429 static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1430 {
1431         switch (port) {
1432         case PORT_B:
1433                 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1434         case PORT_C:
1435                 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1436         case PORT_D:
1437                 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1438         default:
1439                 return false;
1440         }
1441 }
1442
1443 /*
1444  * Get a bit mask of pins that have triggered, and which ones may be long.
1445  * This can be called multiple times with the same masks to accumulate
1446  * hotplug detection results from several registers.
1447  *
1448  * Note that the caller is expected to zero out the masks initially.
1449  */
1450 static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1451                              u32 hotplug_trigger, u32 dig_hotplug_reg,
1452                              const u32 hpd[HPD_NUM_PINS],
1453                              bool long_pulse_detect(enum port port, u32 val))
1454 {
1455         enum port port;
1456         int i;
1457
1458         for_each_hpd_pin(i) {
1459                 if ((hpd[i] & hotplug_trigger) == 0)
1460                         continue;
1461
1462                 *pin_mask |= BIT(i);
1463
1464                 if (!intel_hpd_pin_to_port(i, &port))
1465                         continue;
1466
1467                 if (long_pulse_detect(port, dig_hotplug_reg))
1468                         *long_mask |= BIT(i);
1469         }
1470
1471         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
1472                          hotplug_trigger, dig_hotplug_reg, *pin_mask);
1473
1474 }
1475
1476 static void gmbus_irq_handler(struct drm_device *dev)
1477 {
1478         struct drm_i915_private *dev_priv = dev->dev_private;
1479
1480         wake_up_all(&dev_priv->gmbus_wait_queue);
1481 }
1482
1483 static void dp_aux_irq_handler(struct drm_device *dev)
1484 {
1485         struct drm_i915_private *dev_priv = dev->dev_private;
1486
1487         wake_up_all(&dev_priv->gmbus_wait_queue);
1488 }
1489
1490 #if defined(CONFIG_DEBUG_FS)
1491 static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1492                                          uint32_t crc0, uint32_t crc1,
1493                                          uint32_t crc2, uint32_t crc3,
1494                                          uint32_t crc4)
1495 {
1496         struct drm_i915_private *dev_priv = dev->dev_private;
1497         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1498         struct intel_pipe_crc_entry *entry;
1499         int head, tail;
1500
1501         spin_lock(&pipe_crc->lock);
1502
1503         if (!pipe_crc->entries) {
1504                 spin_unlock(&pipe_crc->lock);
1505                 DRM_DEBUG_KMS("spurious interrupt\n");
1506                 return;
1507         }
1508
1509         head = pipe_crc->head;
1510         tail = pipe_crc->tail;
1511
1512         if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1513                 spin_unlock(&pipe_crc->lock);
1514                 DRM_ERROR("CRC buffer overflowing\n");
1515                 return;
1516         }
1517
1518         entry = &pipe_crc->entries[head];
1519
1520         entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1521         entry->crc[0] = crc0;
1522         entry->crc[1] = crc1;
1523         entry->crc[2] = crc2;
1524         entry->crc[3] = crc3;
1525         entry->crc[4] = crc4;
1526
1527         head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1528         pipe_crc->head = head;
1529
1530         spin_unlock(&pipe_crc->lock);
1531
1532         wake_up_interruptible(&pipe_crc->wq);
1533 }
1534 #else
1535 static inline void
1536 display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1537                              uint32_t crc0, uint32_t crc1,
1538                              uint32_t crc2, uint32_t crc3,
1539                              uint32_t crc4) {}
1540 #endif
1541
1542
1543 static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1544 {
1545         struct drm_i915_private *dev_priv = dev->dev_private;
1546
1547         display_pipe_crc_irq_handler(dev, pipe,
1548                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1549                                      0, 0, 0, 0);
1550 }
1551
1552 static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1553 {
1554         struct drm_i915_private *dev_priv = dev->dev_private;
1555
1556         display_pipe_crc_irq_handler(dev, pipe,
1557                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1558                                      I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1559                                      I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1560                                      I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1561                                      I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1562 }
1563
1564 static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1565 {
1566         struct drm_i915_private *dev_priv = dev->dev_private;
1567         uint32_t res1, res2;
1568
1569         if (INTEL_INFO(dev)->gen >= 3)
1570                 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1571         else
1572                 res1 = 0;
1573
1574         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
1575                 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1576         else
1577                 res2 = 0;
1578
1579         display_pipe_crc_irq_handler(dev, pipe,
1580                                      I915_READ(PIPE_CRC_RES_RED(pipe)),
1581                                      I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1582                                      I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1583                                      res1, res2);
1584 }
1585
1586 /* The RPS events need forcewake, so we add them to a work queue and mask their
1587  * IMR bits until the work is done. Other interrupts can be processed without
1588  * the work queue. */
1589 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1590 {
1591         if (pm_iir & dev_priv->pm_rps_events) {
1592                 spin_lock(&dev_priv->irq_lock);
1593                 gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
1594                 if (dev_priv->rps.interrupts_enabled) {
1595                         dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1596                         queue_work(dev_priv->wq, &dev_priv->rps.work);
1597                 }
1598                 spin_unlock(&dev_priv->irq_lock);
1599         }
1600
1601         if (INTEL_INFO(dev_priv)->gen >= 8)
1602                 return;
1603
1604         if (HAS_VEBOX(dev_priv->dev)) {
1605                 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1606                         notify_ring(&dev_priv->ring[VECS]);
1607
1608                 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1609                         DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1610         }
1611 }
1612
1613 static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
1614 {
1615         if (!drm_handle_vblank(dev, pipe))
1616                 return false;
1617
1618         return true;
1619 }
1620
1621 static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
1622 {
1623         struct drm_i915_private *dev_priv = dev->dev_private;
1624         u32 pipe_stats[I915_MAX_PIPES] = { };
1625         int pipe;
1626
1627         spin_lock(&dev_priv->irq_lock);
1628         for_each_pipe(dev_priv, pipe) {
1629                 int reg;
1630                 u32 mask, iir_bit = 0;
1631
1632                 /*
1633                  * PIPESTAT bits get signalled even when the interrupt is
1634                  * disabled with the mask bits, and some of the status bits do
1635                  * not generate interrupts at all (like the underrun bit). Hence
1636                  * we need to be careful that we only handle what we want to
1637                  * handle.
1638                  */
1639
1640                 /* fifo underruns are filterered in the underrun handler. */
1641                 mask = PIPE_FIFO_UNDERRUN_STATUS;
1642
1643                 switch (pipe) {
1644                 case PIPE_A:
1645                         iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1646                         break;
1647                 case PIPE_B:
1648                         iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1649                         break;
1650                 case PIPE_C:
1651                         iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1652                         break;
1653                 }
1654                 if (iir & iir_bit)
1655                         mask |= dev_priv->pipestat_irq_mask[pipe];
1656
1657                 if (!mask)
1658                         continue;
1659
1660                 reg = PIPESTAT(pipe);
1661                 mask |= PIPESTAT_INT_ENABLE_MASK;
1662                 pipe_stats[pipe] = I915_READ(reg) & mask;
1663
1664                 /*
1665                  * Clear the PIPE*STAT regs before the IIR
1666                  */
1667                 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
1668                                         PIPESTAT_INT_STATUS_MASK))
1669                         I915_WRITE(reg, pipe_stats[pipe]);
1670         }
1671         spin_unlock(&dev_priv->irq_lock);
1672
1673         for_each_pipe(dev_priv, pipe) {
1674                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
1675                     intel_pipe_handle_vblank(dev, pipe))
1676                         intel_check_page_flip(dev, pipe);
1677
1678                 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1679                         intel_prepare_page_flip(dev, pipe);
1680                         intel_finish_page_flip(dev, pipe);
1681                 }
1682
1683                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1684                         i9xx_pipe_crc_irq_handler(dev, pipe);
1685
1686                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1687                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1688         }
1689
1690         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1691                 gmbus_irq_handler(dev);
1692 }
1693
1694 static void i9xx_hpd_irq_handler(struct drm_device *dev)
1695 {
1696         struct drm_i915_private *dev_priv = dev->dev_private;
1697         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1698         u32 pin_mask = 0, long_mask = 0;
1699
1700         if (!hotplug_status)
1701                 return;
1702
1703         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1704         /*
1705          * Make sure hotplug status is cleared before we clear IIR, or else we
1706          * may miss hotplug events.
1707          */
1708         POSTING_READ(PORT_HOTPLUG_STAT);
1709
1710         if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
1711                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1712
1713                 if (hotplug_trigger) {
1714                         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1715                                            hotplug_trigger, hpd_status_g4x,
1716                                            i9xx_port_hotplug_long_detect);
1717
1718                         intel_hpd_irq_handler(dev, pin_mask, long_mask);
1719                 }
1720
1721                 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1722                         dp_aux_irq_handler(dev);
1723         } else {
1724                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1725
1726                 if (hotplug_trigger) {
1727                         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1728                                            hotplug_trigger, hpd_status_i915,
1729                                            i9xx_port_hotplug_long_detect);
1730                         intel_hpd_irq_handler(dev, pin_mask, long_mask);
1731                 }
1732         }
1733 }
1734
1735 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1736 {
1737         struct drm_device *dev = arg;
1738         struct drm_i915_private *dev_priv = dev->dev_private;
1739         u32 iir, gt_iir, pm_iir;
1740         irqreturn_t ret = IRQ_NONE;
1741
1742         if (!intel_irqs_enabled(dev_priv))
1743                 return IRQ_NONE;
1744
1745         while (true) {
1746                 /* Find, clear, then process each source of interrupt */
1747
1748                 gt_iir = I915_READ(GTIIR);
1749                 if (gt_iir)
1750                         I915_WRITE(GTIIR, gt_iir);
1751
1752                 pm_iir = I915_READ(GEN6_PMIIR);
1753                 if (pm_iir)
1754                         I915_WRITE(GEN6_PMIIR, pm_iir);
1755
1756                 iir = I915_READ(VLV_IIR);
1757                 if (iir) {
1758                         /* Consume port before clearing IIR or we'll miss events */
1759                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
1760                                 i9xx_hpd_irq_handler(dev);
1761                         I915_WRITE(VLV_IIR, iir);
1762                 }
1763
1764                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1765                         goto out;
1766
1767                 ret = IRQ_HANDLED;
1768
1769                 if (gt_iir)
1770                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
1771                 if (pm_iir)
1772                         gen6_rps_irq_handler(dev_priv, pm_iir);
1773                 /* Call regardless, as some status bits might not be
1774                  * signalled in iir */
1775                 valleyview_pipestat_irq_handler(dev, iir);
1776         }
1777
1778 out:
1779         return ret;
1780 }
1781
1782 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1783 {
1784         struct drm_device *dev = arg;
1785         struct drm_i915_private *dev_priv = dev->dev_private;
1786         u32 master_ctl, iir;
1787         irqreturn_t ret = IRQ_NONE;
1788
1789         if (!intel_irqs_enabled(dev_priv))
1790                 return IRQ_NONE;
1791
1792         for (;;) {
1793                 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1794                 iir = I915_READ(VLV_IIR);
1795
1796                 if (master_ctl == 0 && iir == 0)
1797                         break;
1798
1799                 ret = IRQ_HANDLED;
1800
1801                 I915_WRITE(GEN8_MASTER_IRQ, 0);
1802
1803                 /* Find, clear, then process each source of interrupt */
1804
1805                 if (iir) {
1806                         /* Consume port before clearing IIR or we'll miss events */
1807                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
1808                                 i9xx_hpd_irq_handler(dev);
1809                         I915_WRITE(VLV_IIR, iir);
1810                 }
1811
1812                 gen8_gt_irq_handler(dev_priv, master_ctl);
1813
1814                 /* Call regardless, as some status bits might not be
1815                  * signalled in iir */
1816                 valleyview_pipestat_irq_handler(dev, iir);
1817
1818                 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
1819                 POSTING_READ(GEN8_MASTER_IRQ);
1820         }
1821
1822         return ret;
1823 }
1824
1825 static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
1826                                 const u32 hpd[HPD_NUM_PINS])
1827 {
1828         struct drm_i915_private *dev_priv = to_i915(dev);
1829         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1830
1831         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1832         I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1833
1834         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1835                            dig_hotplug_reg, hpd,
1836                            pch_port_hotplug_long_detect);
1837
1838         intel_hpd_irq_handler(dev, pin_mask, long_mask);
1839 }
1840
1841 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1842 {
1843         struct drm_i915_private *dev_priv = dev->dev_private;
1844         int pipe;
1845         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1846
1847         if (hotplug_trigger)
1848                 ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1849
1850         if (pch_iir & SDE_AUDIO_POWER_MASK) {
1851                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1852                                SDE_AUDIO_POWER_SHIFT);
1853                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1854                                  port_name(port));
1855         }
1856
1857         if (pch_iir & SDE_AUX_MASK)
1858                 dp_aux_irq_handler(dev);
1859
1860         if (pch_iir & SDE_GMBUS)
1861                 gmbus_irq_handler(dev);
1862
1863         if (pch_iir & SDE_AUDIO_HDCP_MASK)
1864                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1865
1866         if (pch_iir & SDE_AUDIO_TRANS_MASK)
1867                 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1868
1869         if (pch_iir & SDE_POISON)
1870                 DRM_ERROR("PCH poison interrupt\n");
1871
1872         if (pch_iir & SDE_FDI_MASK)
1873                 for_each_pipe(dev_priv, pipe)
1874                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1875                                          pipe_name(pipe),
1876                                          I915_READ(FDI_RX_IIR(pipe)));
1877
1878         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1879                 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1880
1881         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1882                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1883
1884         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1885                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1886
1887         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1888                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1889 }
1890
1891 static void ivb_err_int_handler(struct drm_device *dev)
1892 {
1893         struct drm_i915_private *dev_priv = dev->dev_private;
1894         u32 err_int = I915_READ(GEN7_ERR_INT);
1895         enum pipe pipe;
1896
1897         if (err_int & ERR_INT_POISON)
1898                 DRM_ERROR("Poison interrupt\n");
1899
1900         for_each_pipe(dev_priv, pipe) {
1901                 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1902                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1903
1904                 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1905                         if (IS_IVYBRIDGE(dev))
1906                                 ivb_pipe_crc_irq_handler(dev, pipe);
1907                         else
1908                                 hsw_pipe_crc_irq_handler(dev, pipe);
1909                 }
1910         }
1911
1912         I915_WRITE(GEN7_ERR_INT, err_int);
1913 }
1914
1915 static void cpt_serr_int_handler(struct drm_device *dev)
1916 {
1917         struct drm_i915_private *dev_priv = dev->dev_private;
1918         u32 serr_int = I915_READ(SERR_INT);
1919
1920         if (serr_int & SERR_INT_POISON)
1921                 DRM_ERROR("PCH poison interrupt\n");
1922
1923         if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1924                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1925
1926         if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1927                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1928
1929         if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1930                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
1931
1932         I915_WRITE(SERR_INT, serr_int);
1933 }
1934
1935 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1936 {
1937         struct drm_i915_private *dev_priv = dev->dev_private;
1938         int pipe;
1939         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1940
1941         if (hotplug_trigger)
1942                 ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1943
1944         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1945                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1946                                SDE_AUDIO_POWER_SHIFT_CPT);
1947                 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1948                                  port_name(port));
1949         }
1950
1951         if (pch_iir & SDE_AUX_MASK_CPT)
1952                 dp_aux_irq_handler(dev);
1953
1954         if (pch_iir & SDE_GMBUS_CPT)
1955                 gmbus_irq_handler(dev);
1956
1957         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1958                 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1959
1960         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1961                 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1962
1963         if (pch_iir & SDE_FDI_MASK_CPT)
1964                 for_each_pipe(dev_priv, pipe)
1965                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1966                                          pipe_name(pipe),
1967                                          I915_READ(FDI_RX_IIR(pipe)));
1968
1969         if (pch_iir & SDE_ERROR_CPT)
1970                 cpt_serr_int_handler(dev);
1971 }
1972
1973 static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
1974 {
1975         struct drm_i915_private *dev_priv = dev->dev_private;
1976         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
1977                 ~SDE_PORTE_HOTPLUG_SPT;
1978         u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
1979         u32 pin_mask = 0, long_mask = 0;
1980
1981         if (hotplug_trigger) {
1982                 u32 dig_hotplug_reg;
1983
1984                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1985                 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1986
1987                 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1988                                    dig_hotplug_reg, hpd_spt,
1989                                    spt_port_hotplug_long_detect);
1990         }
1991
1992         if (hotplug2_trigger) {
1993                 u32 dig_hotplug_reg;
1994
1995                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
1996                 I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
1997
1998                 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
1999                                    dig_hotplug_reg, hpd_spt,
2000                                    spt_port_hotplug2_long_detect);
2001         }
2002
2003         if (pin_mask)
2004                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
2005
2006         if (pch_iir & SDE_GMBUS_CPT)
2007                 gmbus_irq_handler(dev);
2008 }
2009
2010 static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
2011                                 const u32 hpd[HPD_NUM_PINS])
2012 {
2013         struct drm_i915_private *dev_priv = to_i915(dev);
2014         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2015
2016         dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
2017         I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
2018
2019         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2020                            dig_hotplug_reg, hpd,
2021                            ilk_port_hotplug_long_detect);
2022
2023         intel_hpd_irq_handler(dev, pin_mask, long_mask);
2024 }
2025
2026 static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
2027 {
2028         struct drm_i915_private *dev_priv = dev->dev_private;
2029         enum pipe pipe;
2030         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
2031
2032         if (hotplug_trigger)
2033                 ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
2034
2035         if (de_iir & DE_AUX_CHANNEL_A)
2036                 dp_aux_irq_handler(dev);
2037
2038         if (de_iir & DE_GSE)
2039                 intel_opregion_asle_intr(dev);
2040
2041         if (de_iir & DE_POISON)
2042                 DRM_ERROR("Poison interrupt\n");
2043
2044         for_each_pipe(dev_priv, pipe) {
2045                 if (de_iir & DE_PIPE_VBLANK(pipe) &&
2046                     intel_pipe_handle_vblank(dev, pipe))
2047                         intel_check_page_flip(dev, pipe);
2048
2049                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2050                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2051
2052                 if (de_iir & DE_PIPE_CRC_DONE(pipe))
2053                         i9xx_pipe_crc_irq_handler(dev, pipe);
2054
2055                 /* plane/pipes map 1:1 on ilk+ */
2056                 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
2057                         intel_prepare_page_flip(dev, pipe);
2058                         intel_finish_page_flip_plane(dev, pipe);
2059                 }
2060         }
2061
2062         /* check event from PCH */
2063         if (de_iir & DE_PCH_EVENT) {
2064                 u32 pch_iir = I915_READ(SDEIIR);
2065
2066                 if (HAS_PCH_CPT(dev))
2067                         cpt_irq_handler(dev, pch_iir);
2068                 else
2069                         ibx_irq_handler(dev, pch_iir);
2070
2071                 /* should clear PCH hotplug event before clear CPU irq */
2072                 I915_WRITE(SDEIIR, pch_iir);
2073         }
2074
2075         if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
2076                 ironlake_rps_change_irq_handler(dev);
2077 }
2078
2079 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
2080 {
2081         struct drm_i915_private *dev_priv = dev->dev_private;
2082         enum pipe pipe;
2083         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2084
2085         if (hotplug_trigger)
2086                 ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
2087
2088         if (de_iir & DE_ERR_INT_IVB)
2089                 ivb_err_int_handler(dev);
2090
2091         if (de_iir & DE_AUX_CHANNEL_A_IVB)
2092                 dp_aux_irq_handler(dev);
2093
2094         if (de_iir & DE_GSE_IVB)
2095                 intel_opregion_asle_intr(dev);
2096
2097         for_each_pipe(dev_priv, pipe) {
2098                 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
2099                     intel_pipe_handle_vblank(dev, pipe))
2100                         intel_check_page_flip(dev, pipe);
2101
2102                 /* plane/pipes map 1:1 on ilk+ */
2103                 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
2104                         intel_prepare_page_flip(dev, pipe);
2105                         intel_finish_page_flip_plane(dev, pipe);
2106                 }
2107         }
2108
2109         /* check event from PCH */
2110         if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
2111                 u32 pch_iir = I915_READ(SDEIIR);
2112
2113                 cpt_irq_handler(dev, pch_iir);
2114
2115                 /* clear PCH hotplug event before clear CPU irq */
2116                 I915_WRITE(SDEIIR, pch_iir);
2117         }
2118 }
2119
2120 /*
2121  * To handle irqs with the minimum potential races with fresh interrupts, we:
2122  * 1 - Disable Master Interrupt Control.
2123  * 2 - Find the source(s) of the interrupt.
2124  * 3 - Clear the Interrupt Identity bits (IIR).
2125  * 4 - Process the interrupt(s) that had bits set in the IIRs.
2126  * 5 - Re-enable Master Interrupt Control.
2127  */
2128 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2129 {
2130         struct drm_device *dev = arg;
2131         struct drm_i915_private *dev_priv = dev->dev_private;
2132         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2133         irqreturn_t ret = IRQ_NONE;
2134
2135         if (!intel_irqs_enabled(dev_priv))
2136                 return IRQ_NONE;
2137
2138         /* We get interrupts on unclaimed registers, so check for this before we
2139          * do any I915_{READ,WRITE}. */
2140         intel_uncore_check_errors(dev);
2141
2142         /* disable master interrupt before clearing iir  */
2143         de_ier = I915_READ(DEIER);
2144         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2145         POSTING_READ(DEIER);
2146
2147         /* Disable south interrupts. We'll only write to SDEIIR once, so further
2148          * interrupts will will be stored on its back queue, and then we'll be
2149          * able to process them after we restore SDEIER (as soon as we restore
2150          * it, we'll get an interrupt if SDEIIR still has something to process
2151          * due to its back queue). */
2152         if (!HAS_PCH_NOP(dev)) {
2153                 sde_ier = I915_READ(SDEIER);
2154                 I915_WRITE(SDEIER, 0);
2155                 POSTING_READ(SDEIER);
2156         }
2157
2158         /* Find, clear, then process each source of interrupt */
2159
2160         gt_iir = I915_READ(GTIIR);
2161         if (gt_iir) {
2162                 I915_WRITE(GTIIR, gt_iir);
2163                 ret = IRQ_HANDLED;
2164                 if (INTEL_INFO(dev)->gen >= 6)
2165                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
2166                 else
2167                         ilk_gt_irq_handler(dev, dev_priv, gt_iir);
2168         }
2169
2170         de_iir = I915_READ(DEIIR);
2171         if (de_iir) {
2172                 I915_WRITE(DEIIR, de_iir);
2173                 ret = IRQ_HANDLED;
2174                 if (INTEL_INFO(dev)->gen >= 7)
2175                         ivb_display_irq_handler(dev, de_iir);
2176                 else
2177                         ilk_display_irq_handler(dev, de_iir);
2178         }
2179
2180         if (INTEL_INFO(dev)->gen >= 6) {
2181                 u32 pm_iir = I915_READ(GEN6_PMIIR);
2182                 if (pm_iir) {
2183                         I915_WRITE(GEN6_PMIIR, pm_iir);
2184                         ret = IRQ_HANDLED;
2185                         gen6_rps_irq_handler(dev_priv, pm_iir);
2186                 }
2187         }
2188
2189         I915_WRITE(DEIER, de_ier);
2190         POSTING_READ(DEIER);
2191         if (!HAS_PCH_NOP(dev)) {
2192                 I915_WRITE(SDEIER, sde_ier);
2193                 POSTING_READ(SDEIER);
2194         }
2195
2196         return ret;
2197 }
2198
2199 static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
2200                                 const u32 hpd[HPD_NUM_PINS])
2201 {
2202         struct drm_i915_private *dev_priv = to_i915(dev);
2203         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2204
2205         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2206         I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2207
2208         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2209                            dig_hotplug_reg, hpd,
2210                            bxt_port_hotplug_long_detect);
2211
2212         intel_hpd_irq_handler(dev, pin_mask, long_mask);
2213 }
2214
2215 static irqreturn_t gen8_irq_handler(int irq, void *arg)
2216 {
2217         struct drm_device *dev = arg;
2218         struct drm_i915_private *dev_priv = dev->dev_private;
2219         u32 master_ctl;
2220         irqreturn_t ret = IRQ_NONE;
2221         uint32_t tmp = 0;
2222         enum pipe pipe;
2223         u32 aux_mask = GEN8_AUX_CHANNEL_A;
2224
2225         if (!intel_irqs_enabled(dev_priv))
2226                 return IRQ_NONE;
2227
2228         if (INTEL_INFO(dev_priv)->gen >= 9)
2229                 aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
2230                         GEN9_AUX_CHANNEL_D;
2231
2232         master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
2233         master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2234         if (!master_ctl)
2235                 return IRQ_NONE;
2236
2237         I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
2238
2239         /* Find, clear, then process each source of interrupt */
2240
2241         ret = gen8_gt_irq_handler(dev_priv, master_ctl);
2242
2243         if (master_ctl & GEN8_DE_MISC_IRQ) {
2244                 tmp = I915_READ(GEN8_DE_MISC_IIR);
2245                 if (tmp) {
2246                         I915_WRITE(GEN8_DE_MISC_IIR, tmp);
2247                         ret = IRQ_HANDLED;
2248                         if (tmp & GEN8_DE_MISC_GSE)
2249                                 intel_opregion_asle_intr(dev);
2250                         else
2251                                 DRM_ERROR("Unexpected DE Misc interrupt\n");
2252                 }
2253                 else
2254                         DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2255         }
2256
2257         if (master_ctl & GEN8_DE_PORT_IRQ) {
2258                 tmp = I915_READ(GEN8_DE_PORT_IIR);
2259                 if (tmp) {
2260                         bool found = false;
2261                         u32 hotplug_trigger = 0;
2262
2263                         if (IS_BROXTON(dev_priv))
2264                                 hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK;
2265                         else if (IS_BROADWELL(dev_priv))
2266                                 hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG;
2267
2268                         I915_WRITE(GEN8_DE_PORT_IIR, tmp);
2269                         ret = IRQ_HANDLED;
2270
2271                         if (tmp & aux_mask) {
2272                                 dp_aux_irq_handler(dev);
2273                                 found = true;
2274                         }
2275
2276                         if (hotplug_trigger) {
2277                                 if (IS_BROXTON(dev))
2278                                         bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt);
2279                                 else
2280                                         ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw);
2281                                 found = true;
2282                         }
2283
2284                         if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
2285                                 gmbus_irq_handler(dev);
2286                                 found = true;
2287                         }
2288
2289                         if (!found)
2290                                 DRM_ERROR("Unexpected DE Port interrupt\n");
2291                 }
2292                 else
2293                         DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2294         }
2295
2296         for_each_pipe(dev_priv, pipe) {
2297                 uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2298
2299                 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2300                         continue;
2301
2302                 pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2303                 if (pipe_iir) {
2304                         ret = IRQ_HANDLED;
2305                         I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2306
2307                         if (pipe_iir & GEN8_PIPE_VBLANK &&
2308                             intel_pipe_handle_vblank(dev, pipe))
2309                                 intel_check_page_flip(dev, pipe);
2310
2311                         if (INTEL_INFO(dev_priv)->gen >= 9)
2312                                 flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
2313                         else
2314                                 flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
2315
2316                         if (flip_done) {
2317                                 intel_prepare_page_flip(dev, pipe);
2318                                 intel_finish_page_flip_plane(dev, pipe);
2319                         }
2320
2321                         if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
2322                                 hsw_pipe_crc_irq_handler(dev, pipe);
2323
2324                         if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
2325                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
2326                                                                     pipe);
2327
2328
2329                         if (INTEL_INFO(dev_priv)->gen >= 9)
2330                                 fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2331                         else
2332                                 fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2333
2334                         if (fault_errors)
2335                                 DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
2336                                           pipe_name(pipe),
2337                                           pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2338                 } else
2339                         DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2340         }
2341
2342         if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
2343             master_ctl & GEN8_DE_PCH_IRQ) {
2344                 /*
2345                  * FIXME(BDW): Assume for now that the new interrupt handling
2346                  * scheme also closed the SDE interrupt handling race we've seen
2347                  * on older pch-split platforms. But this needs testing.
2348                  */
2349                 u32 pch_iir = I915_READ(SDEIIR);
2350                 if (pch_iir) {
2351                         I915_WRITE(SDEIIR, pch_iir);
2352                         ret = IRQ_HANDLED;
2353
2354                         if (HAS_PCH_SPT(dev_priv))
2355                                 spt_irq_handler(dev, pch_iir);
2356                         else
2357                                 cpt_irq_handler(dev, pch_iir);
2358                 } else
2359                         DRM_ERROR("The master control interrupt lied (SDE)!\n");
2360
2361         }
2362
2363         I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2364         POSTING_READ_FW(GEN8_MASTER_IRQ);
2365
2366         return ret;
2367 }
2368
2369 static void i915_error_wake_up(struct drm_i915_private *dev_priv,
2370                                bool reset_completed)
2371 {
2372         struct intel_engine_cs *ring;
2373         int i;
2374
2375         /*
2376          * Notify all waiters for GPU completion events that reset state has
2377          * been changed, and that they need to restart their wait after
2378          * checking for potential errors (and bail out to drop locks if there is
2379          * a gpu reset pending so that i915_error_work_func can acquire them).
2380          */
2381
2382         /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2383         for_each_ring(ring, dev_priv, i)
2384                 wake_up_all(&ring->irq_queue);
2385
2386         /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
2387         wake_up_all(&dev_priv->pending_flip_queue);
2388
2389         /*
2390          * Signal tasks blocked in i915_gem_wait_for_error that the pending
2391          * reset state is cleared.
2392          */
2393         if (reset_completed)
2394                 wake_up_all(&dev_priv->gpu_error.reset_queue);
2395 }
2396
2397 /**
2398  * i915_reset_and_wakeup - do process context error handling work
2399  * @dev: drm device
2400  *
2401  * Fire an error uevent so userspace can see that a hang or error
2402  * was detected.
2403  */
2404 static void i915_reset_and_wakeup(struct drm_device *dev)
2405 {
2406         struct drm_i915_private *dev_priv = to_i915(dev);
2407         struct i915_gpu_error *error = &dev_priv->gpu_error;
2408         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2409         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
2410         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2411         int ret;
2412
2413         kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2414
2415         /*
2416          * Note that there's only one work item which does gpu resets, so we
2417          * need not worry about concurrent gpu resets potentially incrementing
2418          * error->reset_counter twice. We only need to take care of another
2419          * racing irq/hangcheck declaring the gpu dead for a second time. A
2420          * quick check for that is good enough: schedule_work ensures the
2421          * correct ordering between hang detection and this work item, and since
2422          * the reset in-progress bit is only ever set by code outside of this
2423          * work we don't need to worry about any other races.
2424          */
2425         if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2426                 DRM_DEBUG_DRIVER("resetting chip\n");
2427                 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2428                                    reset_event);
2429
2430                 /*
2431                  * In most cases it's guaranteed that we get here with an RPM
2432                  * reference held, for example because there is a pending GPU
2433                  * request that won't finish until the reset is done. This
2434                  * isn't the case at least when we get here by doing a
2435                  * simulated reset via debugs, so get an RPM reference.
2436                  */
2437                 intel_runtime_pm_get(dev_priv);
2438
2439                 intel_prepare_reset(dev);
2440
2441                 /*
2442                  * All state reset _must_ be completed before we update the
2443                  * reset counter, for otherwise waiters might miss the reset
2444                  * pending state and not properly drop locks, resulting in
2445                  * deadlocks with the reset work.
2446                  */
2447                 ret = i915_reset(dev);
2448
2449                 intel_finish_reset(dev);
2450
2451                 intel_runtime_pm_put(dev_priv);
2452
2453                 if (ret == 0) {
2454                         /*
2455                          * After all the gem state is reset, increment the reset
2456                          * counter and wake up everyone waiting for the reset to
2457                          * complete.
2458                          *
2459                          * Since unlock operations are a one-sided barrier only,
2460                          * we need to insert a barrier here to order any seqno
2461                          * updates before
2462                          * the counter increment.
2463                          */
2464                         smp_mb__before_atomic();
2465                         atomic_inc(&dev_priv->gpu_error.reset_counter);
2466
2467                         kobject_uevent_env(&dev->primary->kdev->kobj,
2468                                            KOBJ_CHANGE, reset_done_event);
2469                 } else {
2470                         atomic_or(I915_WEDGED, &error->reset_counter);
2471                 }
2472
2473                 /*
2474                  * Note: The wake_up also serves as a memory barrier so that
2475                  * waiters see the update value of the reset counter atomic_t.
2476                  */
2477                 i915_error_wake_up(dev_priv, true);
2478         }
2479 }
2480
2481 static void i915_report_and_clear_eir(struct drm_device *dev)
2482 {
2483         struct drm_i915_private *dev_priv = dev->dev_private;
2484         uint32_t instdone[I915_NUM_INSTDONE_REG];
2485         u32 eir = I915_READ(EIR);
2486         int pipe, i;
2487
2488         if (!eir)
2489                 return;
2490
2491         pr_err("render error detected, EIR: 0x%08x\n", eir);
2492
2493         i915_get_extra_instdone(dev, instdone);
2494
2495         if (IS_G4X(dev)) {
2496                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
2497                         u32 ipeir = I915_READ(IPEIR_I965);
2498
2499                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2500                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2501                         for (i = 0; i < ARRAY_SIZE(instdone); i++)
2502                                 pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2503                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2504                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2505                         I915_WRITE(IPEIR_I965, ipeir);
2506                         POSTING_READ(IPEIR_I965);
2507                 }
2508                 if (eir & GM45_ERROR_PAGE_TABLE) {
2509                         u32 pgtbl_err = I915_READ(PGTBL_ER);
2510                         pr_err("page table error\n");
2511                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2512                         I915_WRITE(PGTBL_ER, pgtbl_err);
2513                         POSTING_READ(PGTBL_ER);
2514                 }
2515         }
2516
2517         if (!IS_GEN2(dev)) {
2518                 if (eir & I915_ERROR_PAGE_TABLE) {
2519                         u32 pgtbl_err = I915_READ(PGTBL_ER);
2520                         pr_err("page table error\n");
2521                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2522                         I915_WRITE(PGTBL_ER, pgtbl_err);
2523                         POSTING_READ(PGTBL_ER);
2524                 }
2525         }
2526
2527         if (eir & I915_ERROR_MEMORY_REFRESH) {
2528                 pr_err("memory refresh error:\n");
2529                 for_each_pipe(dev_priv, pipe)
2530                         pr_err("pipe %c stat: 0x%08x\n",
2531                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2532                 /* pipestat has already been acked */
2533         }
2534         if (eir & I915_ERROR_INSTRUCTION) {
2535                 pr_err("instruction error\n");
2536                 pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2537                 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2538                         pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2539                 if (INTEL_INFO(dev)->gen < 4) {
2540                         u32 ipeir = I915_READ(IPEIR);
2541
2542                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
2543                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
2544                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2545                         I915_WRITE(IPEIR, ipeir);
2546                         POSTING_READ(IPEIR);
2547                 } else {
2548                         u32 ipeir = I915_READ(IPEIR_I965);
2549
2550                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2551                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2552                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2553                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2554                         I915_WRITE(IPEIR_I965, ipeir);
2555                         POSTING_READ(IPEIR_I965);
2556                 }
2557         }
2558
2559         I915_WRITE(EIR, eir);
2560         POSTING_READ(EIR);
2561         eir = I915_READ(EIR);
2562         if (eir) {
2563                 /*
2564                  * some errors might have become stuck,
2565                  * mask them.
2566                  */
2567                 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2568                 I915_WRITE(EMR, I915_READ(EMR) | eir);
2569                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2570         }
2571 }
2572
2573 /**
2574  * i915_handle_error - handle a gpu error
2575  * @dev: drm device
2576  *
2577  * Do some basic checking of register state at error time and
2578  * dump it to the syslog.  Also call i915_capture_error_state() to make
2579  * sure we get a record and make it available in debugfs.  Fire a uevent
2580  * so userspace knows something bad happened (should trigger collection
2581  * of a ring dump etc.).
2582  */
2583 void i915_handle_error(struct drm_device *dev, bool wedged,
2584                        const char *fmt, ...)
2585 {
2586         struct drm_i915_private *dev_priv = dev->dev_private;
2587         va_list args;
2588         char error_msg[80];
2589
2590         va_start(args, fmt);
2591         vscnprintf(error_msg, sizeof(error_msg), fmt, args);
2592         va_end(args);
2593
2594         i915_capture_error_state(dev, wedged, error_msg);
2595         i915_report_and_clear_eir(dev);
2596
2597         if (wedged) {
2598                 atomic_or(I915_RESET_IN_PROGRESS_FLAG,
2599                                 &dev_priv->gpu_error.reset_counter);
2600
2601                 /*
2602                  * Wakeup waiting processes so that the reset function
2603                  * i915_reset_and_wakeup doesn't deadlock trying to grab
2604                  * various locks. By bumping the reset counter first, the woken
2605                  * processes will see a reset in progress and back off,
2606                  * releasing their locks and then wait for the reset completion.
2607                  * We must do this for _all_ gpu waiters that might hold locks
2608                  * that the reset work needs to acquire.
2609                  *
2610                  * Note: The wake_up serves as the required memory barrier to
2611                  * ensure that the waiters see the updated value of the reset
2612                  * counter atomic_t.
2613                  */
2614                 i915_error_wake_up(dev_priv, false);
2615         }
2616
2617         i915_reset_and_wakeup(dev);
2618 }
2619
2620 /* Called from drm generic code, passed 'crtc' which
2621  * we use as a pipe index
2622  */
2623 static int i915_enable_vblank(struct drm_device *dev, int pipe)
2624 {
2625         struct drm_i915_private *dev_priv = dev->dev_private;
2626         unsigned long irqflags;
2627
2628         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2629         if (INTEL_INFO(dev)->gen >= 4)
2630                 i915_enable_pipestat(dev_priv, pipe,
2631                                      PIPE_START_VBLANK_INTERRUPT_STATUS);
2632         else
2633                 i915_enable_pipestat(dev_priv, pipe,
2634                                      PIPE_VBLANK_INTERRUPT_STATUS);
2635         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2636
2637         return 0;
2638 }
2639
2640 static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2641 {
2642         struct drm_i915_private *dev_priv = dev->dev_private;
2643         unsigned long irqflags;
2644         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2645                                                      DE_PIPE_VBLANK(pipe);
2646
2647         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2648         ironlake_enable_display_irq(dev_priv, bit);
2649         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2650
2651         return 0;
2652 }
2653
2654 static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
2655 {
2656         struct drm_i915_private *dev_priv = dev->dev_private;
2657         unsigned long irqflags;
2658
2659         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2660         i915_enable_pipestat(dev_priv, pipe,
2661                              PIPE_START_VBLANK_INTERRUPT_STATUS);
2662         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2663
2664         return 0;
2665 }
2666
2667 static int gen8_enable_vblank(struct drm_device *dev, int pipe)
2668 {
2669         struct drm_i915_private *dev_priv = dev->dev_private;
2670         unsigned long irqflags;
2671
2672         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2673         dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
2674         I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2675         POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2676         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2677         return 0;
2678 }
2679
2680 /* Called from drm generic code, passed 'crtc' which
2681  * we use as a pipe index
2682  */
2683 static void i915_disable_vblank(struct drm_device *dev, int pipe)
2684 {
2685         struct drm_i915_private *dev_priv = dev->dev_private;
2686         unsigned long irqflags;
2687
2688         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2689         i915_disable_pipestat(dev_priv, pipe,
2690                               PIPE_VBLANK_INTERRUPT_STATUS |
2691                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2692         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2693 }
2694
2695 static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2696 {
2697         struct drm_i915_private *dev_priv = dev->dev_private;
2698         unsigned long irqflags;
2699         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2700                                                      DE_PIPE_VBLANK(pipe);
2701
2702         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2703         ironlake_disable_display_irq(dev_priv, bit);
2704         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2705 }
2706
2707 static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
2708 {
2709         struct drm_i915_private *dev_priv = dev->dev_private;
2710         unsigned long irqflags;
2711
2712         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2713         i915_disable_pipestat(dev_priv, pipe,
2714                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2715         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2716 }
2717
2718 static void gen8_disable_vblank(struct drm_device *dev, int pipe)
2719 {
2720         struct drm_i915_private *dev_priv = dev->dev_private;
2721         unsigned long irqflags;
2722
2723         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2724         dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
2725         I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2726         POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2727         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2728 }
2729
2730 static bool
2731 ring_idle(struct intel_engine_cs *ring, u32 seqno)
2732 {
2733         return (list_empty(&ring->request_list) ||
2734                 i915_seqno_passed(seqno, ring->last_submitted_seqno));
2735 }
2736
2737 static bool
2738 ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
2739 {
2740         if (INTEL_INFO(dev)->gen >= 8) {
2741                 return (ipehr >> 23) == 0x1c;
2742         } else {
2743                 ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
2744                 return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
2745                                  MI_SEMAPHORE_REGISTER);
2746         }
2747 }
2748
2749 static struct intel_engine_cs *
2750 semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2751 {
2752         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2753         struct intel_engine_cs *signaller;
2754         int i;
2755
2756         if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2757                 for_each_ring(signaller, dev_priv, i) {
2758                         if (ring == signaller)
2759                                 continue;
2760
2761                         if (offset == signaller->semaphore.signal_ggtt[ring->id])
2762                                 return signaller;
2763                 }
2764         } else {
2765                 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
2766
2767                 for_each_ring(signaller, dev_priv, i) {
2768                         if(ring == signaller)
2769                                 continue;
2770
2771                         if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2772                                 return signaller;
2773                 }
2774         }
2775
2776         DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2777                   ring->id, ipehr, offset);
2778
2779         return NULL;
2780 }
2781
2782 static struct intel_engine_cs *
2783 semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2784 {
2785         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2786         u32 cmd, ipehr, head;
2787         u64 offset = 0;
2788         int i, backwards;
2789
2790         /*
2791          * This function does not support execlist mode - any attempt to
2792          * proceed further into this function will result in a kernel panic
2793          * when dereferencing ring->buffer, which is not set up in execlist
2794          * mode.
2795          *
2796          * The correct way of doing it would be to derive the currently
2797          * executing ring buffer from the current context, which is derived
2798          * from the currently running request. Unfortunately, to get the
2799          * current request we would have to grab the struct_mutex before doing
2800          * anything else, which would be ill-advised since some other thread
2801          * might have grabbed it already and managed to hang itself, causing
2802          * the hang checker to deadlock.
2803          *
2804          * Therefore, this function does not support execlist mode in its
2805          * current form. Just return NULL and move on.
2806          */
2807         if (ring->buffer == NULL)
2808                 return NULL;
2809
2810         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2811         if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2812                 return NULL;
2813
2814         /*
2815          * HEAD is likely pointing to the dword after the actual command,
2816          * so scan backwards until we find the MBOX. But limit it to just 3
2817          * or 4 dwords depending on the semaphore wait command size.
2818          * Note that we don't care about ACTHD here since that might
2819          * point at at batch, and semaphores are always emitted into the
2820          * ringbuffer itself.
2821          */
2822         head = I915_READ_HEAD(ring) & HEAD_ADDR;
2823         backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2824
2825         for (i = backwards; i; --i) {
2826                 /*
2827                  * Be paranoid and presume the hw has gone off into the wild -
2828                  * our ring is smaller than what the hardware (and hence
2829                  * HEAD_ADDR) allows. Also handles wrap-around.
2830                  */
2831                 head &= ring->buffer->size - 1;
2832
2833                 /* This here seems to blow up */
2834                 cmd = ioread32(ring->buffer->virtual_start + head);
2835                 if (cmd == ipehr)
2836                         break;
2837
2838                 head -= 4;
2839         }
2840
2841         if (!i)
2842                 return NULL;
2843
2844         *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2845         if (INTEL_INFO(ring->dev)->gen >= 8) {
2846                 offset = ioread32(ring->buffer->virtual_start + head + 12);
2847                 offset <<= 32;
2848                 offset = ioread32(ring->buffer->virtual_start + head + 8);
2849         }
2850         return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2851 }
2852
2853 static int semaphore_passed(struct intel_engine_cs *ring)
2854 {
2855         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2856         struct intel_engine_cs *signaller;
2857         u32 seqno;
2858
2859         ring->hangcheck.deadlock++;
2860
2861         signaller = semaphore_waits_for(ring, &seqno);
2862         if (signaller == NULL)
2863                 return -1;
2864
2865         /* Prevent pathological recursion due to driver bugs */
2866         if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2867                 return -1;
2868
2869         if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
2870                 return 1;
2871
2872         /* cursory check for an unkickable deadlock */
2873         if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
2874             semaphore_passed(signaller) < 0)
2875                 return -1;
2876
2877         return 0;
2878 }
2879
2880 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
2881 {
2882         struct intel_engine_cs *ring;
2883         int i;
2884
2885         for_each_ring(ring, dev_priv, i)
2886                 ring->hangcheck.deadlock = 0;
2887 }
2888
2889 static enum intel_ring_hangcheck_action
2890 ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2891 {
2892         struct drm_device *dev = ring->dev;
2893         struct drm_i915_private *dev_priv = dev->dev_private;
2894         u32 tmp;
2895
2896         if (acthd != ring->hangcheck.acthd) {
2897                 if (acthd > ring->hangcheck.max_acthd) {
2898                         ring->hangcheck.max_acthd = acthd;
2899                         return HANGCHECK_ACTIVE;
2900                 }
2901
2902                 return HANGCHECK_ACTIVE_LOOP;
2903         }
2904
2905         if (IS_GEN2(dev))
2906                 return HANGCHECK_HUNG;
2907
2908         /* Is the chip hanging on a WAIT_FOR_EVENT?
2909          * If so we can simply poke the RB_WAIT bit
2910          * and break the hang. This should work on
2911          * all but the second generation chipsets.
2912          */
2913         tmp = I915_READ_CTL(ring);
2914         if (tmp & RING_WAIT) {
2915                 i915_handle_error(dev, false,
2916                                   "Kicking stuck wait on %s",
2917                                   ring->name);
2918                 I915_WRITE_CTL(ring, tmp);
2919                 return HANGCHECK_KICK;
2920         }
2921
2922         if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
2923                 switch (semaphore_passed(ring)) {
2924                 default:
2925                         return HANGCHECK_HUNG;
2926                 case 1:
2927                         i915_handle_error(dev, false,
2928                                           "Kicking stuck semaphore on %s",
2929                                           ring->name);
2930                         I915_WRITE_CTL(ring, tmp);
2931                         return HANGCHECK_KICK;
2932                 case 0:
2933                         return HANGCHECK_WAIT;
2934                 }
2935         }
2936
2937         return HANGCHECK_HUNG;
2938 }
2939
2940 /*
2941  * This is called when the chip hasn't reported back with completed
2942  * batchbuffers in a long time. We keep track per ring seqno progress and
2943  * if there are no progress, hangcheck score for that ring is increased.
2944  * Further, acthd is inspected to see if the ring is stuck. On stuck case
2945  * we kick the ring. If we see no progress on three subsequent calls
2946  * we assume chip is wedged and try to fix it by resetting the chip.
2947  */
2948 static void i915_hangcheck_elapsed(struct work_struct *work)
2949 {
2950         struct drm_i915_private *dev_priv =
2951                 container_of(work, typeof(*dev_priv),
2952                              gpu_error.hangcheck_work.work);
2953         struct drm_device *dev = dev_priv->dev;
2954         struct intel_engine_cs *ring;
2955         int i;
2956         int busy_count = 0, rings_hung = 0;
2957         bool stuck[I915_NUM_RINGS] = { 0 };
2958 #define BUSY 1
2959 #define KICK 5
2960 #define HUNG 20
2961
2962         if (!i915.enable_hangcheck)
2963                 return;
2964
2965         for_each_ring(ring, dev_priv, i) {
2966                 u64 acthd;
2967                 u32 seqno;
2968                 bool busy = true;
2969
2970                 semaphore_clear_deadlocks(dev_priv);
2971
2972                 seqno = ring->get_seqno(ring, false);
2973                 acthd = intel_ring_get_active_head(ring);
2974
2975                 if (ring->hangcheck.seqno == seqno) {
2976                         if (ring_idle(ring, seqno)) {
2977                                 ring->hangcheck.action = HANGCHECK_IDLE;
2978
2979                                 if (waitqueue_active(&ring->irq_queue)) {
2980                                         /* Issue a wake-up to catch stuck h/w. */
2981                                         if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2982                                                 if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
2983                                                         DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2984                                                                   ring->name);
2985                                                 else
2986                                                         DRM_INFO("Fake missed irq on %s\n",
2987                                                                  ring->name);
2988                                                 wake_up_all(&ring->irq_queue);
2989                                         }
2990                                         /* Safeguard against driver failure */
2991                                         ring->hangcheck.score += BUSY;
2992                                 } else
2993                                         busy = false;
2994                         } else {
2995                                 /* We always increment the hangcheck score
2996                                  * if the ring is busy and still processing
2997                                  * the same request, so that no single request
2998                                  * can run indefinitely (such as a chain of
2999                                  * batches). The only time we do not increment
3000                                  * the hangcheck score on this ring, if this
3001                                  * ring is in a legitimate wait for another
3002                                  * ring. In that case the waiting ring is a
3003                                  * victim and we want to be sure we catch the
3004                                  * right culprit. Then every time we do kick
3005                                  * the ring, add a small increment to the
3006                                  * score so that we can catch a batch that is
3007                                  * being repeatedly kicked and so responsible
3008                                  * for stalling the machine.
3009                                  */
3010                                 ring->hangcheck.action = ring_stuck(ring,
3011                                                                     acthd);
3012
3013                                 switch (ring->hangcheck.action) {
3014                                 case HANGCHECK_IDLE:
3015                                 case HANGCHECK_WAIT:
3016                                 case HANGCHECK_ACTIVE:
3017                                         break;
3018                                 case HANGCHECK_ACTIVE_LOOP:
3019                                         ring->hangcheck.score += BUSY;
3020                                         break;
3021                                 case HANGCHECK_KICK:
3022                                         ring->hangcheck.score += KICK;
3023                                         break;
3024                                 case HANGCHECK_HUNG:
3025                                         ring->hangcheck.score += HUNG;
3026                                         stuck[i] = true;
3027                                         break;
3028                                 }
3029                         }
3030                 } else {
3031                         ring->hangcheck.action = HANGCHECK_ACTIVE;
3032
3033                         /* Gradually reduce the count so that we catch DoS
3034                          * attempts across multiple batches.
3035                          */
3036                         if (ring->hangcheck.score > 0)
3037                                 ring->hangcheck.score--;
3038
3039                         ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
3040                 }
3041
3042                 ring->hangcheck.seqno = seqno;
3043                 ring->hangcheck.acthd = acthd;
3044                 busy_count += busy;
3045         }
3046
3047         for_each_ring(ring, dev_priv, i) {
3048                 if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
3049                         DRM_INFO("%s on %s\n",
3050                                  stuck[i] ? "stuck" : "no progress",
3051                                  ring->name);
3052                         rings_hung++;
3053                 }
3054         }
3055
3056         if (rings_hung)
3057                 return i915_handle_error(dev, true, "Ring hung");
3058
3059         if (busy_count)
3060                 /* Reset timer case chip hangs without another request
3061                  * being added */
3062                 i915_queue_hangcheck(dev);
3063 }
3064
3065 void i915_queue_hangcheck(struct drm_device *dev)
3066 {
3067         struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
3068
3069         if (!i915.enable_hangcheck)
3070                 return;
3071
3072         /* Don't continually defer the hangcheck so that it is always run at
3073          * least once after work has been scheduled on any ring. Otherwise,
3074          * we will ignore a hung ring if a second ring is kept busy.
3075          */
3076
3077         queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
3078                            round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
3079 }
3080
3081 static void ibx_irq_reset(struct drm_device *dev)
3082 {
3083         struct drm_i915_private *dev_priv = dev->dev_private;
3084
3085         if (HAS_PCH_NOP(dev))
3086                 return;
3087
3088         GEN5_IRQ_RESET(SDE);
3089
3090         if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
3091                 I915_WRITE(SERR_INT, 0xffffffff);
3092 }
3093
3094 /*
3095  * SDEIER is also touched by the interrupt handler to work around missed PCH
3096  * interrupts. Hence we can't update it after the interrupt handler is enabled -
3097  * instead we unconditionally enable all PCH interrupt sources here, but then
3098  * only unmask them as needed with SDEIMR.
3099  *
3100  * This function needs to be called before interrupts are enabled.
3101  */
3102 static void ibx_irq_pre_postinstall(struct drm_device *dev)
3103 {
3104         struct drm_i915_private *dev_priv = dev->dev_private;
3105
3106         if (HAS_PCH_NOP(dev))
3107                 return;
3108
3109         WARN_ON(I915_READ(SDEIER) != 0);
3110         I915_WRITE(SDEIER, 0xffffffff);
3111         POSTING_READ(SDEIER);
3112 }
3113
3114 static void gen5_gt_irq_reset(struct drm_device *dev)
3115 {
3116         struct drm_i915_private *dev_priv = dev->dev_private;
3117
3118         GEN5_IRQ_RESET(GT);
3119         if (INTEL_INFO(dev)->gen >= 6)
3120                 GEN5_IRQ_RESET(GEN6_PM);
3121 }
3122
3123 /* drm_dma.h hooks
3124 */
3125 static void ironlake_irq_reset(struct drm_device *dev)
3126 {
3127         struct drm_i915_private *dev_priv = dev->dev_private;
3128
3129         I915_WRITE(HWSTAM, 0xffffffff);
3130
3131         GEN5_IRQ_RESET(DE);
3132         if (IS_GEN7(dev))
3133                 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3134
3135         gen5_gt_irq_reset(dev);
3136
3137         ibx_irq_reset(dev);
3138 }
3139
3140 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
3141 {
3142         enum pipe pipe;
3143
3144         i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0);
3145         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3146
3147         for_each_pipe(dev_priv, pipe)
3148                 I915_WRITE(PIPESTAT(pipe), 0xffff);
3149
3150         GEN5_IRQ_RESET(VLV_);
3151 }
3152
3153 static void valleyview_irq_preinstall(struct drm_device *dev)
3154 {
3155         struct drm_i915_private *dev_priv = dev->dev_private;
3156
3157         /* VLV magic */
3158         I915_WRITE(VLV_IMR, 0);
3159         I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
3160         I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
3161         I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
3162
3163         gen5_gt_irq_reset(dev);
3164
3165         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3166
3167         vlv_display_irq_reset(dev_priv);
3168 }
3169
3170 static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
3171 {
3172         GEN8_IRQ_RESET_NDX(GT, 0);
3173         GEN8_IRQ_RESET_NDX(GT, 1);
3174         GEN8_IRQ_RESET_NDX(GT, 2);
3175         GEN8_IRQ_RESET_NDX(GT, 3);
3176 }
3177
3178 static void gen8_irq_reset(struct drm_device *dev)
3179 {
3180         struct drm_i915_private *dev_priv = dev->dev_private;
3181         int pipe;
3182
3183         I915_WRITE(GEN8_MASTER_IRQ, 0);
3184         POSTING_READ(GEN8_MASTER_IRQ);
3185
3186         gen8_gt_irq_reset(dev_priv);
3187
3188         for_each_pipe(dev_priv, pipe)
3189                 if (intel_display_power_is_enabled(dev_priv,
3190                                                    POWER_DOMAIN_PIPE(pipe)))
3191                         GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3192
3193         GEN5_IRQ_RESET(GEN8_DE_PORT_);
3194         GEN5_IRQ_RESET(GEN8_DE_MISC_);
3195         GEN5_IRQ_RESET(GEN8_PCU_);
3196
3197         if (HAS_PCH_SPLIT(dev))
3198                 ibx_irq_reset(dev);
3199 }
3200
3201 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
3202                                      unsigned int pipe_mask)
3203 {
3204         uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3205
3206         spin_lock_irq(&dev_priv->irq_lock);
3207         if (pipe_mask & 1 << PIPE_A)
3208                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
3209                                   dev_priv->de_irq_mask[PIPE_A],
3210                                   ~dev_priv->de_irq_mask[PIPE_A] | extra_ier);
3211         if (pipe_mask & 1 << PIPE_B)
3212                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
3213                                   dev_priv->de_irq_mask[PIPE_B],
3214                                   ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
3215         if (pipe_mask & 1 << PIPE_C)
3216                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
3217                                   dev_priv->de_irq_mask[PIPE_C],
3218                                   ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
3219         spin_unlock_irq(&dev_priv->irq_lock);
3220 }
3221
3222 static void cherryview_irq_preinstall(struct drm_device *dev)
3223 {
3224         struct drm_i915_private *dev_priv = dev->dev_private;
3225
3226         I915_WRITE(GEN8_MASTER_IRQ, 0);
3227         POSTING_READ(GEN8_MASTER_IRQ);
3228
3229         gen8_gt_irq_reset(dev_priv);
3230
3231         GEN5_IRQ_RESET(GEN8_PCU_);
3232
3233         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
3234
3235         vlv_display_irq_reset(dev_priv);
3236 }
3237
3238 static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
3239                                   const u32 hpd[HPD_NUM_PINS])
3240 {
3241         struct drm_i915_private *dev_priv = to_i915(dev);
3242         struct intel_encoder *encoder;
3243         u32 enabled_irqs = 0;
3244
3245         for_each_intel_encoder(dev, encoder)
3246                 if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
3247                         enabled_irqs |= hpd[encoder->hpd_pin];
3248
3249         return enabled_irqs;
3250 }
3251
3252 static void ibx_hpd_irq_setup(struct drm_device *dev)
3253 {
3254         struct drm_i915_private *dev_priv = dev->dev_private;
3255         u32 hotplug_irqs, hotplug, enabled_irqs;
3256
3257         if (HAS_PCH_IBX(dev)) {
3258                 hotplug_irqs = SDE_HOTPLUG_MASK;
3259                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
3260         } else {
3261                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3262                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
3263         }
3264
3265         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3266
3267         /*
3268          * Enable digital hotplug on the PCH, and configure the DP short pulse
3269          * duration to 2ms (which is the minimum in the Display Port spec).
3270          * The pulse duration bits are reserved on LPT+.
3271          */
3272         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3273         hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
3274         hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3275         hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
3276         hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3277         /*
3278          * When CPU and PCH are on the same package, port A
3279          * HPD must be enabled in both north and south.
3280          */
3281         if (HAS_PCH_LPT_LP(dev))
3282                 hotplug |= PORTA_HOTPLUG_ENABLE;
3283         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3284 }
3285
3286 static void spt_hpd_irq_setup(struct drm_device *dev)
3287 {
3288         struct drm_i915_private *dev_priv = dev->dev_private;
3289         u32 hotplug_irqs, hotplug, enabled_irqs;
3290
3291         hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
3292         enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);
3293
3294         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3295
3296         /* Enable digital hotplug on the PCH */
3297         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3298         hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE |
3299                 PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE;
3300         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3301
3302         hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3303         hotplug |= PORTE_HOTPLUG_ENABLE;
3304         I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3305 }
3306
3307 static void ilk_hpd_irq_setup(struct drm_device *dev)
3308 {
3309         struct drm_i915_private *dev_priv = dev->dev_private;
3310         u32 hotplug_irqs, hotplug, enabled_irqs;
3311
3312         if (INTEL_INFO(dev)->gen >= 8) {
3313                 hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3314                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);
3315
3316                 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3317         } else if (INTEL_INFO(dev)->gen >= 7) {
3318                 hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3319                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
3320
3321                 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3322         } else {
3323                 hotplug_irqs = DE_DP_A_HOTPLUG;
3324                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
3325
3326                 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3327         }
3328
3329         /*
3330          * Enable digital hotplug on the CPU, and configure the DP short pulse
3331          * duration to 2ms (which is the minimum in the Display Port spec)
3332          * The pulse duration bits are reserved on HSW+.
3333          */
3334         hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
3335         hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
3336         hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
3337         I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
3338
3339         ibx_hpd_irq_setup(dev);
3340 }
3341
3342 static void bxt_hpd_irq_setup(struct drm_device *dev)
3343 {
3344         struct drm_i915_private *dev_priv = dev->dev_private;
3345         u32 hotplug_irqs, hotplug, enabled_irqs;
3346
3347         enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
3348         hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3349
3350         bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3351
3352         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3353         hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE |
3354                 PORTA_HOTPLUG_ENABLE;
3355         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3356 }
3357
3358 static void ibx_irq_postinstall(struct drm_device *dev)
3359 {
3360         struct drm_i915_private *dev_priv = dev->dev_private;
3361         u32 mask;
3362
3363         if (HAS_PCH_NOP(dev))
3364                 return;
3365
3366         if (HAS_PCH_IBX(dev))
3367                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3368         else
3369                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3370
3371         gen5_assert_iir_is_zero(dev_priv, SDEIIR);
3372         I915_WRITE(SDEIMR, ~mask);
3373 }
3374
3375 static void gen5_gt_irq_postinstall(struct drm_device *dev)
3376 {
3377         struct drm_i915_private *dev_priv = dev->dev_private;
3378         u32 pm_irqs, gt_irqs;
3379
3380         pm_irqs = gt_irqs = 0;
3381
3382         dev_priv->gt_irq_mask = ~0;
3383         if (HAS_L3_DPF(dev)) {
3384                 /* L3 parity interrupt is always unmasked. */
3385                 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
3386                 gt_irqs |= GT_PARITY_ERROR(dev);
3387         }
3388
3389         gt_irqs |= GT_RENDER_USER_INTERRUPT;
3390         if (IS_GEN5(dev)) {
3391                 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
3392                            ILK_BSD_USER_INTERRUPT;
3393         } else {
3394                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
3395         }
3396
3397         GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3398
3399         if (INTEL_INFO(dev)->gen >= 6) {
3400                 /*
3401                  * RPS interrupts will get enabled/disabled on demand when RPS
3402                  * itself is enabled/disabled.
3403                  */
3404                 if (HAS_VEBOX(dev))
3405                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3406
3407                 dev_priv->pm_irq_mask = 0xffffffff;
3408                 GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3409         }
3410 }
3411
3412 static int ironlake_irq_postinstall(struct drm_device *dev)
3413 {
3414         struct drm_i915_private *dev_priv = dev->dev_private;
3415         u32 display_mask, extra_mask;
3416
3417         if (INTEL_INFO(dev)->gen >= 7) {
3418                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3419                                 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
3420                                 DE_PLANEB_FLIP_DONE_IVB |
3421                                 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3422                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3423                               DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
3424                               DE_DP_A_HOTPLUG_IVB);
3425         } else {
3426                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3427                                 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3428                                 DE_AUX_CHANNEL_A |
3429                                 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
3430                                 DE_POISON);
3431                 extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3432                               DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
3433                               DE_DP_A_HOTPLUG);
3434         }
3435
3436         dev_priv->irq_mask = ~display_mask;
3437
3438         I915_WRITE(HWSTAM, 0xeffe);
3439
3440         ibx_irq_pre_postinstall(dev);
3441
3442         GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3443
3444         gen5_gt_irq_postinstall(dev);
3445
3446         ibx_irq_postinstall(dev);
3447
3448         if (IS_IRONLAKE_M(dev)) {
3449                 /* Enable PCU event interrupts
3450                  *
3451                  * spinlocking not required here for correctness since interrupt
3452                  * setup is guaranteed to run in single-threaded context. But we
3453                  * need it to make the assert_spin_locked happy. */
3454                 spin_lock_irq(&dev_priv->irq_lock);
3455                 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3456                 spin_unlock_irq(&dev_priv->irq_lock);
3457         }
3458
3459         return 0;
3460 }
3461
3462 static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
3463 {
3464         u32 pipestat_mask;
3465         u32 iir_mask;
3466         enum pipe pipe;
3467
3468         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3469                         PIPE_FIFO_UNDERRUN_STATUS;
3470
3471         for_each_pipe(dev_priv, pipe)
3472                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3473         POSTING_READ(PIPESTAT(PIPE_A));
3474
3475         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3476                         PIPE_CRC_DONE_INTERRUPT_STATUS;
3477
3478         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3479         for_each_pipe(dev_priv, pipe)
3480                       i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3481
3482         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3483                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3484                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3485         if (IS_CHERRYVIEW(dev_priv))
3486                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3487         dev_priv->irq_mask &= ~iir_mask;
3488
3489         I915_WRITE(VLV_IIR, iir_mask);
3490         I915_WRITE(VLV_IIR, iir_mask);
3491         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3492         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3493         POSTING_READ(VLV_IMR);
3494 }
3495
3496 static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
3497 {
3498         u32 pipestat_mask;
3499         u32 iir_mask;
3500         enum pipe pipe;
3501
3502         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3503                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3504                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3505         if (IS_CHERRYVIEW(dev_priv))
3506                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3507
3508         dev_priv->irq_mask |= iir_mask;
3509         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3510         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3511         I915_WRITE(VLV_IIR, iir_mask);
3512         I915_WRITE(VLV_IIR, iir_mask);
3513         POSTING_READ(VLV_IIR);
3514
3515         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3516                         PIPE_CRC_DONE_INTERRUPT_STATUS;
3517
3518         i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3519         for_each_pipe(dev_priv, pipe)
3520                 i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3521
3522         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3523                         PIPE_FIFO_UNDERRUN_STATUS;
3524
3525         for_each_pipe(dev_priv, pipe)
3526                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3527         POSTING_READ(PIPESTAT(PIPE_A));
3528 }
3529
3530 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3531 {
3532         assert_spin_locked(&dev_priv->irq_lock);
3533
3534         if (dev_priv->display_irqs_enabled)
3535                 return;
3536
3537         dev_priv->display_irqs_enabled = true;
3538
3539         if (intel_irqs_enabled(dev_priv))
3540                 valleyview_display_irqs_install(dev_priv);
3541 }
3542
3543 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3544 {
3545         assert_spin_locked(&dev_priv->irq_lock);
3546
3547         if (!dev_priv->display_irqs_enabled)
3548                 return;
3549
3550         dev_priv->display_irqs_enabled = false;
3551
3552         if (intel_irqs_enabled(dev_priv))
3553                 valleyview_display_irqs_uninstall(dev_priv);
3554 }
3555
3556 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3557 {
3558         dev_priv->irq_mask = ~0;
3559
3560         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3561         POSTING_READ(PORT_HOTPLUG_EN);
3562
3563         I915_WRITE(VLV_IIR, 0xffffffff);
3564         I915_WRITE(VLV_IIR, 0xffffffff);
3565         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3566         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3567         POSTING_READ(VLV_IMR);
3568
3569         /* Interrupt setup is already guaranteed to be single-threaded, this is
3570          * just to make the assert_spin_locked check happy. */
3571         spin_lock_irq(&dev_priv->irq_lock);
3572         if (dev_priv->display_irqs_enabled)
3573                 valleyview_display_irqs_install(dev_priv);
3574         spin_unlock_irq(&dev_priv->irq_lock);
3575 }
3576
3577 static int valleyview_irq_postinstall(struct drm_device *dev)
3578 {
3579         struct drm_i915_private *dev_priv = dev->dev_private;
3580
3581         vlv_display_irq_postinstall(dev_priv);
3582
3583         gen5_gt_irq_postinstall(dev);
3584
3585         /* ack & enable invalid PTE error interrupts */
3586 #if 0 /* FIXME: add support to irq handler for checking these bits */
3587         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3588         I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
3589 #endif
3590
3591         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3592
3593         return 0;
3594 }
3595
3596 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3597 {
3598         /* These are interrupts we'll toggle with the ring mask register */
3599         uint32_t gt_interrupts[] = {
3600                 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3601                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3602                         GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3603                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
3604                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3605                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3606                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3607                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
3608                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3609                 0,
3610                 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
3611                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3612                 };
3613
3614         dev_priv->pm_irq_mask = 0xffffffff;
3615         GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
3616         GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3617         /*
3618          * RPS interrupts will get enabled/disabled on demand when RPS itself
3619          * is enabled/disabled.
3620          */
3621         GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3622         GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3623 }
3624
3625 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3626 {
3627         uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3628         uint32_t de_pipe_enables;
3629         u32 de_port_masked = GEN8_AUX_CHANNEL_A;
3630         u32 de_port_enables;
3631         enum pipe pipe;
3632
3633         if (INTEL_INFO(dev_priv)->gen >= 9) {
3634                 de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
3635                                   GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3636                 de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3637                                   GEN9_AUX_CHANNEL_D;
3638                 if (IS_BROXTON(dev_priv))
3639                         de_port_masked |= BXT_DE_PORT_GMBUS;
3640         } else {
3641                 de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
3642                                   GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3643         }
3644
3645         de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3646                                            GEN8_PIPE_FIFO_UNDERRUN;
3647
3648         de_port_enables = de_port_masked;
3649         if (IS_BROXTON(dev_priv))
3650                 de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
3651         else if (IS_BROADWELL(dev_priv))
3652                 de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
3653
3654         dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
3655         dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
3656         dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3657
3658         for_each_pipe(dev_priv, pipe)
3659                 if (intel_display_power_is_enabled(dev_priv,
3660                                 POWER_DOMAIN_PIPE(pipe)))
3661                         GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3662                                           dev_priv->de_irq_mask[pipe],
3663                                           de_pipe_enables);
3664
3665         GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3666 }
3667
3668 static int gen8_irq_postinstall(struct drm_device *dev)
3669 {
3670         struct drm_i915_private *dev_priv = dev->dev_private;
3671
3672         if (HAS_PCH_SPLIT(dev))
3673                 ibx_irq_pre_postinstall(dev);
3674
3675         gen8_gt_irq_postinstall(dev_priv);
3676         gen8_de_irq_postinstall(dev_priv);
3677
3678         if (HAS_PCH_SPLIT(dev))
3679                 ibx_irq_postinstall(dev);
3680
3681         I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
3682         POSTING_READ(GEN8_MASTER_IRQ);
3683
3684         return 0;
3685 }
3686
3687 static int cherryview_irq_postinstall(struct drm_device *dev)
3688 {
3689         struct drm_i915_private *dev_priv = dev->dev_private;
3690
3691         vlv_display_irq_postinstall(dev_priv);
3692
3693         gen8_gt_irq_postinstall(dev_priv);
3694
3695         I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
3696         POSTING_READ(GEN8_MASTER_IRQ);
3697
3698         return 0;
3699 }
3700
3701 static void gen8_irq_uninstall(struct drm_device *dev)
3702 {
3703         struct drm_i915_private *dev_priv = dev->dev_private;
3704
3705         if (!dev_priv)
3706                 return;
3707
3708         gen8_irq_reset(dev);
3709 }
3710
3711 static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
3712 {
3713         /* Interrupt setup is already guaranteed to be single-threaded, this is
3714          * just to make the assert_spin_locked check happy. */
3715         spin_lock_irq(&dev_priv->irq_lock);
3716         if (dev_priv->display_irqs_enabled)
3717                 valleyview_display_irqs_uninstall(dev_priv);
3718         spin_unlock_irq(&dev_priv->irq_lock);
3719
3720         vlv_display_irq_reset(dev_priv);
3721
3722         dev_priv->irq_mask = ~0;
3723 }
3724
3725 static void valleyview_irq_uninstall(struct drm_device *dev)
3726 {
3727         struct drm_i915_private *dev_priv = dev->dev_private;
3728
3729         if (!dev_priv)
3730                 return;
3731
3732         I915_WRITE(VLV_MASTER_IER, 0);
3733
3734         gen5_gt_irq_reset(dev);
3735
3736         I915_WRITE(HWSTAM, 0xffffffff);
3737
3738         vlv_display_irq_uninstall(dev_priv);
3739 }
3740
3741 static void cherryview_irq_uninstall(struct drm_device *dev)
3742 {
3743         struct drm_i915_private *dev_priv = dev->dev_private;
3744
3745         if (!dev_priv)
3746                 return;
3747
3748         I915_WRITE(GEN8_MASTER_IRQ, 0);
3749         POSTING_READ(GEN8_MASTER_IRQ);
3750
3751         gen8_gt_irq_reset(dev_priv);
3752
3753         GEN5_IRQ_RESET(GEN8_PCU_);
3754
3755         vlv_display_irq_uninstall(dev_priv);
3756 }
3757
3758 static void ironlake_irq_uninstall(struct drm_device *dev)
3759 {
3760         struct drm_i915_private *dev_priv = dev->dev_private;
3761
3762         if (!dev_priv)
3763                 return;
3764
3765         ironlake_irq_reset(dev);
3766 }
3767
3768 static void i8xx_irq_preinstall(struct drm_device * dev)
3769 {
3770         struct drm_i915_private *dev_priv = dev->dev_private;
3771         int pipe;
3772
3773         for_each_pipe(dev_priv, pipe)
3774                 I915_WRITE(PIPESTAT(pipe), 0);
3775         I915_WRITE16(IMR, 0xffff);
3776         I915_WRITE16(IER, 0x0);
3777         POSTING_READ16(IER);
3778 }
3779
3780 static int i8xx_irq_postinstall(struct drm_device *dev)
3781 {
3782         struct drm_i915_private *dev_priv = dev->dev_private;
3783
3784         I915_WRITE16(EMR,
3785                      ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3786
3787         /* Unmask the interrupts that we always want on. */
3788         dev_priv->irq_mask =
3789                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3790                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3791                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3792                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3793         I915_WRITE16(IMR, dev_priv->irq_mask);
3794
3795         I915_WRITE16(IER,
3796                      I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3797                      I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3798                      I915_USER_INTERRUPT);
3799         POSTING_READ16(IER);
3800
3801         /* Interrupt setup is already guaranteed to be single-threaded, this is
3802          * just to make the assert_spin_locked check happy. */
3803         spin_lock_irq(&dev_priv->irq_lock);
3804         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3805         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3806         spin_unlock_irq(&dev_priv->irq_lock);
3807
3808         return 0;
3809 }
3810
3811 /*
3812  * Returns true when a page flip has completed.
3813  */
3814 static bool i8xx_handle_vblank(struct drm_device *dev,
3815                                int plane, int pipe, u32 iir)
3816 {
3817         struct drm_i915_private *dev_priv = dev->dev_private;
3818         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3819
3820         if (!intel_pipe_handle_vblank(dev, pipe))
3821                 return false;
3822
3823         if ((iir & flip_pending) == 0)
3824                 goto check_page_flip;
3825
3826         /* We detect FlipDone by looking for the change in PendingFlip from '1'
3827          * to '0' on the following vblank, i.e. IIR has the Pendingflip
3828          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3829          * the flip is completed (no longer pending). Since this doesn't raise
3830          * an interrupt per se, we watch for the change at vblank.
3831          */
3832         if (I915_READ16(ISR) & flip_pending)
3833                 goto check_page_flip;
3834
3835         intel_prepare_page_flip(dev, plane);
3836         intel_finish_page_flip(dev, pipe);
3837         return true;
3838
3839 check_page_flip:
3840         intel_check_page_flip(dev, pipe);
3841         return false;
3842 }
3843
3844 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3845 {
3846         struct drm_device *dev = arg;
3847         struct drm_i915_private *dev_priv = dev->dev_private;
3848         u16 iir, new_iir;
3849         u32 pipe_stats[2];
3850         int pipe;
3851         u16 flip_mask =
3852                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3853                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3854
3855         if (!intel_irqs_enabled(dev_priv))
3856                 return IRQ_NONE;
3857
3858         iir = I915_READ16(IIR);
3859         if (iir == 0)
3860                 return IRQ_NONE;
3861
3862         while (iir & ~flip_mask) {
3863                 /* Can't rely on pipestat interrupt bit in iir as it might
3864                  * have been cleared after the pipestat interrupt was received.
3865                  * It doesn't set the bit in iir again, but it still produces
3866                  * interrupts (for non-MSI).
3867                  */
3868                 spin_lock(&dev_priv->irq_lock);
3869                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3870                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3871
3872                 for_each_pipe(dev_priv, pipe) {
3873                         int reg = PIPESTAT(pipe);
3874                         pipe_stats[pipe] = I915_READ(reg);
3875
3876                         /*
3877                          * Clear the PIPE*STAT regs before the IIR
3878                          */
3879                         if (pipe_stats[pipe] & 0x8000ffff)
3880                                 I915_WRITE(reg, pipe_stats[pipe]);
3881                 }
3882                 spin_unlock(&dev_priv->irq_lock);
3883
3884                 I915_WRITE16(IIR, iir & ~flip_mask);
3885                 new_iir = I915_READ16(IIR); /* Flush posted writes */
3886
3887                 if (iir & I915_USER_INTERRUPT)
3888                         notify_ring(&dev_priv->ring[RCS]);
3889
3890                 for_each_pipe(dev_priv, pipe) {
3891                         int plane = pipe;
3892                         if (HAS_FBC(dev))
3893                                 plane = !plane;
3894
3895                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3896                             i8xx_handle_vblank(dev, plane, pipe, iir))
3897                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3898
3899                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3900                                 i9xx_pipe_crc_irq_handler(dev, pipe);
3901
3902                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3903                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3904                                                                     pipe);
3905                 }
3906
3907                 iir = new_iir;
3908         }
3909
3910         return IRQ_HANDLED;
3911 }
3912
3913 static void i8xx_irq_uninstall(struct drm_device * dev)
3914 {
3915         struct drm_i915_private *dev_priv = dev->dev_private;
3916         int pipe;
3917
3918         for_each_pipe(dev_priv, pipe) {
3919                 /* Clear enable bits; then clear status bits */
3920                 I915_WRITE(PIPESTAT(pipe), 0);
3921                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3922         }
3923         I915_WRITE16(IMR, 0xffff);
3924         I915_WRITE16(IER, 0x0);
3925         I915_WRITE16(IIR, I915_READ16(IIR));
3926 }
3927
3928 static void i915_irq_preinstall(struct drm_device * dev)
3929 {
3930         struct drm_i915_private *dev_priv = dev->dev_private;
3931         int pipe;
3932
3933         if (I915_HAS_HOTPLUG(dev)) {
3934                 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3935                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3936         }
3937
3938         I915_WRITE16(HWSTAM, 0xeffe);
3939         for_each_pipe(dev_priv, pipe)
3940                 I915_WRITE(PIPESTAT(pipe), 0);
3941         I915_WRITE(IMR, 0xffffffff);
3942         I915_WRITE(IER, 0x0);
3943         POSTING_READ(IER);
3944 }
3945
3946 static int i915_irq_postinstall(struct drm_device *dev)
3947 {
3948         struct drm_i915_private *dev_priv = dev->dev_private;
3949         u32 enable_mask;
3950
3951         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3952
3953         /* Unmask the interrupts that we always want on. */
3954         dev_priv->irq_mask =
3955                 ~(I915_ASLE_INTERRUPT |
3956                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3957                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3958                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3959                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3960
3961         enable_mask =
3962                 I915_ASLE_INTERRUPT |
3963                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3964                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3965                 I915_USER_INTERRUPT;
3966
3967         if (I915_HAS_HOTPLUG(dev)) {
3968                 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3969                 POSTING_READ(PORT_HOTPLUG_EN);
3970
3971                 /* Enable in IER... */
3972                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3973                 /* and unmask in IMR */
3974                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3975         }
3976
3977         I915_WRITE(IMR, dev_priv->irq_mask);
3978         I915_WRITE(IER, enable_mask);
3979         POSTING_READ(IER);
3980
3981         i915_enable_asle_pipestat(dev);
3982
3983         /* Interrupt setup is already guaranteed to be single-threaded, this is
3984          * just to make the assert_spin_locked check happy. */
3985         spin_lock_irq(&dev_priv->irq_lock);
3986         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3987         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3988         spin_unlock_irq(&dev_priv->irq_lock);
3989
3990         return 0;
3991 }
3992
3993 /*
3994  * Returns true when a page flip has completed.
3995  */
3996 static bool i915_handle_vblank(struct drm_device *dev,
3997                                int plane, int pipe, u32 iir)
3998 {
3999         struct drm_i915_private *dev_priv = dev->dev_private;
4000         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
4001
4002         if (!intel_pipe_handle_vblank(dev, pipe))
4003                 return false;
4004
4005         if ((iir & flip_pending) == 0)
4006                 goto check_page_flip;
4007
4008         /* We detect FlipDone by looking for the change in PendingFlip from '1'
4009          * to '0' on the following vblank, i.e. IIR has the Pendingflip
4010          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
4011          * the flip is completed (no longer pending). Since this doesn't raise
4012          * an interrupt per se, we watch for the change at vblank.
4013          */
4014         if (I915_READ(ISR) & flip_pending)
4015                 goto check_page_flip;
4016
4017         intel_prepare_page_flip(dev, plane);
4018         intel_finish_page_flip(dev, pipe);
4019         return true;
4020
4021 check_page_flip:
4022         intel_check_page_flip(dev, pipe);
4023         return false;
4024 }
4025
4026 static irqreturn_t i915_irq_handler(int irq, void *arg)
4027 {
4028         struct drm_device *dev = arg;
4029         struct drm_i915_private *dev_priv = dev->dev_private;
4030         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
4031         u32 flip_mask =
4032                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4033                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4034         int pipe, ret = IRQ_NONE;
4035
4036         if (!intel_irqs_enabled(dev_priv))
4037                 return IRQ_NONE;
4038
4039         iir = I915_READ(IIR);
4040         do {
4041                 bool irq_received = (iir & ~flip_mask) != 0;
4042                 bool blc_event = false;
4043
4044                 /* Can't rely on pipestat interrupt bit in iir as it might
4045                  * have been cleared after the pipestat interrupt was received.
4046                  * It doesn't set the bit in iir again, but it still produces
4047                  * interrupts (for non-MSI).
4048                  */
4049                 spin_lock(&dev_priv->irq_lock);
4050                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4051                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4052
4053                 for_each_pipe(dev_priv, pipe) {
4054                         int reg = PIPESTAT(pipe);
4055                         pipe_stats[pipe] = I915_READ(reg);
4056
4057                         /* Clear the PIPE*STAT regs before the IIR */
4058                         if (pipe_stats[pipe] & 0x8000ffff) {
4059                                 I915_WRITE(reg, pipe_stats[pipe]);
4060                                 irq_received = true;
4061                         }
4062                 }
4063                 spin_unlock(&dev_priv->irq_lock);
4064
4065                 if (!irq_received)
4066                         break;
4067
4068                 /* Consume port.  Then clear IIR or we'll miss events */
4069                 if (I915_HAS_HOTPLUG(dev) &&
4070                     iir & I915_DISPLAY_PORT_INTERRUPT)
4071                         i9xx_hpd_irq_handler(dev);
4072
4073                 I915_WRITE(IIR, iir & ~flip_mask);
4074                 new_iir = I915_READ(IIR); /* Flush posted writes */
4075
4076                 if (iir & I915_USER_INTERRUPT)
4077                         notify_ring(&dev_priv->ring[RCS]);
4078
4079                 for_each_pipe(dev_priv, pipe) {
4080                         int plane = pipe;
4081                         if (HAS_FBC(dev))
4082                                 plane = !plane;
4083
4084                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
4085                             i915_handle_vblank(dev, plane, pipe, iir))
4086                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
4087
4088                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4089                                 blc_event = true;
4090
4091                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4092                                 i9xx_pipe_crc_irq_handler(dev, pipe);
4093
4094                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4095                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
4096                                                                     pipe);
4097                 }
4098
4099                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
4100                         intel_opregion_asle_intr(dev);
4101
4102                 /* With MSI, interrupts are only generated when iir
4103                  * transitions from zero to nonzero.  If another bit got
4104                  * set while we were handling the existing iir bits, then
4105                  * we would never get another interrupt.
4106                  *
4107                  * This is fine on non-MSI as well, as if we hit this path
4108                  * we avoid exiting the interrupt handler only to generate
4109                  * another one.
4110                  *
4111                  * Note that for MSI this could cause a stray interrupt report
4112                  * if an interrupt landed in the time between writing IIR and
4113                  * the posting read.  This should be rare enough to never
4114                  * trigger the 99% of 100,000 interrupts test for disabling
4115                  * stray interrupts.
4116                  */
4117                 ret = IRQ_HANDLED;
4118                 iir = new_iir;
4119         } while (iir & ~flip_mask);
4120
4121         return ret;
4122 }
4123
4124 static void i915_irq_uninstall(struct drm_device * dev)
4125 {
4126         struct drm_i915_private *dev_priv = dev->dev_private;
4127         int pipe;
4128
4129         if (I915_HAS_HOTPLUG(dev)) {
4130                 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4131                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4132         }
4133
4134         I915_WRITE16(HWSTAM, 0xffff);
4135         for_each_pipe(dev_priv, pipe) {
4136                 /* Clear enable bits; then clear status bits */
4137                 I915_WRITE(PIPESTAT(pipe), 0);
4138                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
4139         }
4140         I915_WRITE(IMR, 0xffffffff);
4141         I915_WRITE(IER, 0x0);
4142
4143         I915_WRITE(IIR, I915_READ(IIR));
4144 }
4145
4146 static void i965_irq_preinstall(struct drm_device * dev)
4147 {
4148         struct drm_i915_private *dev_priv = dev->dev_private;
4149         int pipe;
4150
4151         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4152         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4153
4154         I915_WRITE(HWSTAM, 0xeffe);
4155         for_each_pipe(dev_priv, pipe)
4156                 I915_WRITE(PIPESTAT(pipe), 0);
4157         I915_WRITE(IMR, 0xffffffff);
4158         I915_WRITE(IER, 0x0);
4159         POSTING_READ(IER);
4160 }
4161
4162 static int i965_irq_postinstall(struct drm_device *dev)
4163 {
4164         struct drm_i915_private *dev_priv = dev->dev_private;
4165         u32 enable_mask;
4166         u32 error_mask;
4167
4168         /* Unmask the interrupts that we always want on. */
4169         dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4170                                I915_DISPLAY_PORT_INTERRUPT |
4171                                I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4172                                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4173                                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4174                                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
4175                                I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
4176
4177         enable_mask = ~dev_priv->irq_mask;
4178         enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4179                          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4180         enable_mask |= I915_USER_INTERRUPT;
4181
4182         if (IS_G4X(dev))
4183                 enable_mask |= I915_BSD_USER_INTERRUPT;
4184
4185         /* Interrupt setup is already guaranteed to be single-threaded, this is
4186          * just to make the assert_spin_locked check happy. */
4187         spin_lock_irq(&dev_priv->irq_lock);
4188         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4189         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4190         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4191         spin_unlock_irq(&dev_priv->irq_lock);
4192
4193         /*
4194          * Enable some error detection, note the instruction error mask
4195          * bit is reserved, so we leave it masked.
4196          */
4197         if (IS_G4X(dev)) {
4198                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
4199                                GM45_ERROR_MEM_PRIV |
4200                                GM45_ERROR_CP_PRIV |
4201                                I915_ERROR_MEMORY_REFRESH);
4202         } else {
4203                 error_mask = ~(I915_ERROR_PAGE_TABLE |
4204                                I915_ERROR_MEMORY_REFRESH);
4205         }
4206         I915_WRITE(EMR, error_mask);
4207
4208         I915_WRITE(IMR, dev_priv->irq_mask);
4209         I915_WRITE(IER, enable_mask);
4210         POSTING_READ(IER);
4211
4212         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4213         POSTING_READ(PORT_HOTPLUG_EN);
4214
4215         i915_enable_asle_pipestat(dev);
4216
4217         return 0;
4218 }
4219
4220 static void i915_hpd_irq_setup(struct drm_device *dev)
4221 {
4222         struct drm_i915_private *dev_priv = dev->dev_private;
4223         u32 hotplug_en;
4224
4225         assert_spin_locked(&dev_priv->irq_lock);
4226
4227         /* Note HDMI and DP share hotplug bits */
4228         /* enable bits are the same for all generations */
4229         hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
4230         /* Programming the CRT detection parameters tends
4231            to generate a spurious hotplug event about three
4232            seconds later.  So just do it once.
4233         */
4234         if (IS_G4X(dev))
4235                 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
4236         hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4237
4238         /* Ignore TV since it's buggy */
4239         i915_hotplug_interrupt_update_locked(dev_priv,
4240                                       (HOTPLUG_INT_EN_MASK
4241                                        | CRT_HOTPLUG_VOLTAGE_COMPARE_MASK),
4242                                       hotplug_en);
4243 }
4244
4245 static irqreturn_t i965_irq_handler(int irq, void *arg)
4246 {
4247         struct drm_device *dev = arg;
4248         struct drm_i915_private *dev_priv = dev->dev_private;
4249         u32 iir, new_iir;
4250         u32 pipe_stats[I915_MAX_PIPES];
4251         int ret = IRQ_NONE, pipe;
4252         u32 flip_mask =
4253                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4254                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4255
4256         if (!intel_irqs_enabled(dev_priv))
4257                 return IRQ_NONE;
4258
4259         iir = I915_READ(IIR);
4260
4261         for (;;) {
4262                 bool irq_received = (iir & ~flip_mask) != 0;
4263                 bool blc_event = false;
4264
4265                 /* Can't rely on pipestat interrupt bit in iir as it might
4266                  * have been cleared after the pipestat interrupt was received.
4267                  * It doesn't set the bit in iir again, but it still produces
4268                  * interrupts (for non-MSI).
4269                  */
4270                 spin_lock(&dev_priv->irq_lock);
4271                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4272                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4273
4274                 for_each_pipe(dev_priv, pipe) {
4275                         int reg = PIPESTAT(pipe);
4276                         pipe_stats[pipe] = I915_READ(reg);
4277
4278                         /*
4279                          * Clear the PIPE*STAT regs before the IIR
4280                          */
4281                         if (pipe_stats[pipe] & 0x8000ffff) {
4282                                 I915_WRITE(reg, pipe_stats[pipe]);
4283                                 irq_received = true;
4284                         }
4285                 }
4286                 spin_unlock(&dev_priv->irq_lock);
4287
4288                 if (!irq_received)
4289                         break;
4290
4291                 ret = IRQ_HANDLED;
4292
4293                 /* Consume port.  Then clear IIR or we'll miss events */
4294                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
4295                         i9xx_hpd_irq_handler(dev);
4296
4297                 I915_WRITE(IIR, iir & ~flip_mask);
4298                 new_iir = I915_READ(IIR); /* Flush posted writes */
4299
4300                 if (iir & I915_USER_INTERRUPT)
4301                         notify_ring(&dev_priv->ring[RCS]);
4302                 if (iir & I915_BSD_USER_INTERRUPT)
4303                         notify_ring(&dev_priv->ring[VCS]);
4304
4305                 for_each_pipe(dev_priv, pipe) {
4306                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4307                             i915_handle_vblank(dev, pipe, pipe, iir))
4308                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4309
4310                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4311                                 blc_event = true;
4312
4313                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4314                                 i9xx_pipe_crc_irq_handler(dev, pipe);
4315
4316                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4317                                 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4318                 }
4319
4320                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
4321                         intel_opregion_asle_intr(dev);
4322
4323                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4324                         gmbus_irq_handler(dev);
4325
4326                 /* With MSI, interrupts are only generated when iir
4327                  * transitions from zero to nonzero.  If another bit got
4328                  * set while we were handling the existing iir bits, then
4329                  * we would never get another interrupt.
4330                  *
4331                  * This is fine on non-MSI as well, as if we hit this path
4332                  * we avoid exiting the interrupt handler only to generate
4333                  * another one.
4334                  *
4335                  * Note that for MSI this could cause a stray interrupt report
4336                  * if an interrupt landed in the time between writing IIR and
4337                  * the posting read.  This should be rare enough to never
4338                  * trigger the 99% of 100,000 interrupts test for disabling
4339                  * stray interrupts.
4340                  */
4341                 iir = new_iir;
4342         }
4343
4344         return ret;
4345 }
4346
4347 static void i965_irq_uninstall(struct drm_device * dev)
4348 {
4349         struct drm_i915_private *dev_priv = dev->dev_private;
4350         int pipe;
4351
4352         if (!dev_priv)
4353                 return;
4354
4355         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4356         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4357
4358         I915_WRITE(HWSTAM, 0xffffffff);
4359         for_each_pipe(dev_priv, pipe)
4360                 I915_WRITE(PIPESTAT(pipe), 0);
4361         I915_WRITE(IMR, 0xffffffff);
4362         I915_WRITE(IER, 0x0);
4363
4364         for_each_pipe(dev_priv, pipe)
4365                 I915_WRITE(PIPESTAT(pipe),
4366                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
4367         I915_WRITE(IIR, I915_READ(IIR));
4368 }
4369
4370 /**
4371  * intel_irq_init - initializes irq support
4372  * @dev_priv: i915 device instance
4373  *
4374  * This function initializes all the irq support including work items, timers
4375  * and all the vtables. It does not setup the interrupt itself though.
4376  */
4377 void intel_irq_init(struct drm_i915_private *dev_priv)
4378 {
4379         struct drm_device *dev = dev_priv->dev;
4380
4381         intel_hpd_init_work(dev_priv);
4382
4383         INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4384         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4385
4386         /* Let's track the enabled rps events */
4387         if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4388                 /* WaGsvRC0ResidencyMethod:vlv */
4389                 dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4390         else
4391                 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4392
4393         INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
4394                           i915_hangcheck_elapsed);
4395
4396         pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4397
4398         if (IS_GEN2(dev_priv)) {
4399                 dev->max_vblank_count = 0;
4400                 dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4401         } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4402                 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4403                 dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4404         } else {
4405                 dev->driver->get_vblank_counter = i915_get_vblank_counter;
4406                 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4407         }
4408
4409         /*
4410          * Opt out of the vblank disable timer on everything except gen2.
4411          * Gen2 doesn't have a hardware frame counter and so depends on
4412          * vblank interrupts to produce sane vblank seuquence numbers.
4413          */
4414         if (!IS_GEN2(dev_priv))
4415                 dev->vblank_disable_immediate = true;
4416
4417         dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4418         dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4419
4420         if (IS_CHERRYVIEW(dev_priv)) {
4421                 dev->driver->irq_handler = cherryview_irq_handler;
4422                 dev->driver->irq_preinstall = cherryview_irq_preinstall;
4423                 dev->driver->irq_postinstall = cherryview_irq_postinstall;
4424                 dev->driver->irq_uninstall = cherryview_irq_uninstall;
4425                 dev->driver->enable_vblank = valleyview_enable_vblank;
4426                 dev->driver->disable_vblank = valleyview_disable_vblank;
4427                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4428         } else if (IS_VALLEYVIEW(dev_priv)) {
4429                 dev->driver->irq_handler = valleyview_irq_handler;
4430                 dev->driver->irq_preinstall = valleyview_irq_preinstall;
4431                 dev->driver->irq_postinstall = valleyview_irq_postinstall;
4432                 dev->driver->irq_uninstall = valleyview_irq_uninstall;
4433                 dev->driver->enable_vblank = valleyview_enable_vblank;
4434                 dev->driver->disable_vblank = valleyview_disable_vblank;
4435                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4436         } else if (INTEL_INFO(dev_priv)->gen >= 8) {
4437                 dev->driver->irq_handler = gen8_irq_handler;
4438                 dev->driver->irq_preinstall = gen8_irq_reset;
4439                 dev->driver->irq_postinstall = gen8_irq_postinstall;
4440                 dev->driver->irq_uninstall = gen8_irq_uninstall;
4441                 dev->driver->enable_vblank = gen8_enable_vblank;
4442                 dev->driver->disable_vblank = gen8_disable_vblank;
4443                 if (IS_BROXTON(dev))
4444                         dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4445                 else if (HAS_PCH_SPT(dev))
4446                         dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
4447                 else
4448                         dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4449         } else if (HAS_PCH_SPLIT(dev)) {
4450                 dev->driver->irq_handler = ironlake_irq_handler;
4451                 dev->driver->irq_preinstall = ironlake_irq_reset;
4452                 dev->driver->irq_postinstall = ironlake_irq_postinstall;
4453                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
4454                 dev->driver->enable_vblank = ironlake_enable_vblank;
4455                 dev->driver->disable_vblank = ironlake_disable_vblank;
4456                 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4457         } else {
4458                 if (INTEL_INFO(dev_priv)->gen == 2) {
4459                         dev->driver->irq_preinstall = i8xx_irq_preinstall;
4460                         dev->driver->irq_postinstall = i8xx_irq_postinstall;
4461                         dev->driver->irq_handler = i8xx_irq_handler;
4462                         dev->driver->irq_uninstall = i8xx_irq_uninstall;
4463                 } else if (INTEL_INFO(dev_priv)->gen == 3) {
4464                         dev->driver->irq_preinstall = i915_irq_preinstall;
4465                         dev->driver->irq_postinstall = i915_irq_postinstall;
4466                         dev->driver->irq_uninstall = i915_irq_uninstall;
4467                         dev->driver->irq_handler = i915_irq_handler;
4468                 } else {
4469                         dev->driver->irq_preinstall = i965_irq_preinstall;
4470                         dev->driver->irq_postinstall = i965_irq_postinstall;
4471                         dev->driver->irq_uninstall = i965_irq_uninstall;
4472                         dev->driver->irq_handler = i965_irq_handler;
4473                 }
4474                 if (I915_HAS_HOTPLUG(dev_priv))
4475                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4476                 dev->driver->enable_vblank = i915_enable_vblank;
4477                 dev->driver->disable_vblank = i915_disable_vblank;
4478         }
4479 }
4480
4481 /**
4482  * intel_irq_install - enables the hardware interrupt
4483  * @dev_priv: i915 device instance
4484  *
4485  * This function enables the hardware interrupt handling, but leaves the hotplug
4486  * handling still disabled. It is called after intel_irq_init().
4487  *
4488  * In the driver load and resume code we need working interrupts in a few places
4489  * but don't want to deal with the hassle of concurrent probe and hotplug
4490  * workers. Hence the split into this two-stage approach.
4491  */
4492 int intel_irq_install(struct drm_i915_private *dev_priv)
4493 {
4494         /*
4495          * We enable some interrupt sources in our postinstall hooks, so mark
4496          * interrupts as enabled _before_ actually enabling them to avoid
4497          * special cases in our ordering checks.
4498          */
4499         dev_priv->pm.irqs_enabled = true;
4500
4501         return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
4502 }
4503
4504 /**
4505  * intel_irq_uninstall - finilizes all irq handling
4506  * @dev_priv: i915 device instance
4507  *
4508  * This stops interrupt and hotplug handling and unregisters and frees all
4509  * resources acquired in the init functions.
4510  */
4511 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4512 {
4513         drm_irq_uninstall(dev_priv->dev);
4514         intel_hpd_cancel_work(dev_priv);
4515         dev_priv->pm.irqs_enabled = false;
4516 }
4517
4518 /**
4519  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4520  * @dev_priv: i915 device instance
4521  *
4522  * This function is used to disable interrupts at runtime, both in the runtime
4523  * pm and the system suspend/resume code.
4524  */
4525 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4526 {
4527         dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4528         dev_priv->pm.irqs_enabled = false;
4529         synchronize_irq(dev_priv->dev->irq);
4530 }
4531
4532 /**
4533  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4534  * @dev_priv: i915 device instance
4535  *
4536  * This function is used to enable interrupts at runtime, both in the runtime
4537  * pm and the system suspend/resume code.
4538  */
4539 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4540 {
4541         dev_priv->pm.irqs_enabled = true;
4542         dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
4543         dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4544 }