drivers/gpu/drm/i915/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/cpufreq.h>
  28 #include <linux/module.h>
  29 #include <linux/input.h>
  30 #include <linux/i2c.h>
  31 #include <linux/kernel.h>
  32 #include <linux/slab.h>
  33 #include <linux/vgaarb.h>
  34 #include <drm/drm_edid.h>
  35 #include "drmP.h"
  36 #include "intel_drv.h"
  37 #include "i915_drm.h"
  38 #include "i915_drv.h"
  39 #include "i915_trace.h"
  40 #include "drm_dp_helper.h"
  41 #include "drm_crtc_helper.h"
  42 #include <linux/dma_remapping.h>
  43
  44 #define HAS_eDP (intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))
  45
  46 bool intel_pipe_has_type(struct drm_crtc *crtc, int type);
  47 static void intel_update_watermarks(struct drm_device *dev);
  48 static void intel_increase_pllclock(struct drm_crtc *crtc);
  49 static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on);
  50
  51 typedef struct {
  52         /* given values */
  53         int n;
  54         int m1, m2;
  55         int p1, p2;
  56         /* derived values */
  57         int     dot;
  58         int     vco;
  59         int     m;
  60         int     p;
  61 } intel_clock_t;
  62
  63 typedef struct {
  64         int     min, max;
  65 } intel_range_t;
  66
  67 typedef struct {
  68         int     dot_limit;
  69         int     p2_slow, p2_fast;
  70 } intel_p2_t;
  71
  72 #define INTEL_P2_NUM                  2
  73 typedef struct intel_limit intel_limit_t;
  74 struct intel_limit {
  75         intel_range_t   dot, vco, n, m, m1, m2, p, p1;
  76         intel_p2_t          p2;
  77         bool (* find_pll)(const intel_limit_t *, struct drm_crtc *,
  78                         int, int, intel_clock_t *, intel_clock_t *);
  79 };
  80
  81 /* FDI */
  82 #define IRONLAKE_FDI_FREQ               2700000 /* in kHz for mode->clock */
  83
  84 static bool
  85 intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
  86                     int target, int refclk, intel_clock_t *match_clock,
  87                     intel_clock_t *best_clock);
  88 static bool
  89 intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
  90                         int target, int refclk, intel_clock_t *match_clock,
  91                         intel_clock_t *best_clock);
  92
  93 static bool
  94 intel_find_pll_g4x_dp(const intel_limit_t *, struct drm_crtc *crtc,
  95                       int target, int refclk, intel_clock_t *match_clock,
  96                       intel_clock_t *best_clock);
  97 static bool
  98 intel_find_pll_ironlake_dp(const intel_limit_t *, struct drm_crtc *crtc,
  99                            int target, int refclk, intel_clock_t *match_clock,
 100                            intel_clock_t *best_clock);
 101
 102 static inline u32 /* units of 100MHz */
 103 intel_fdi_link_freq(struct drm_device *dev)
 104 {
 105         if (IS_GEN5(dev)) {
 106                 struct drm_i915_private *dev_priv = dev->dev_private;
 107                 return (I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK) + 2;
 108         } else
 109                 return 27;
 110 }
 111
 112 static const intel_limit_t intel_limits_i8xx_dvo = {
 113         .dot = { .min = 25000, .max = 350000 },
 114         .vco = { .min = 930000, .max = 1400000 },
 115         .n = { .min = 3, .max = 16 },
 116         .m = { .min = 96, .max = 140 },
 117         .m1 = { .min = 18, .max = 26 },
 118         .m2 = { .min = 6, .max = 16 },
 119         .p = { .min = 4, .max = 128 },
 120         .p1 = { .min = 2, .max = 33 },
 121         .p2 = { .dot_limit = 165000,
 122                 .p2_slow = 4, .p2_fast = 2 },
 123         .find_pll = intel_find_best_PLL,
 124 };
 125
 126 static const intel_limit_t intel_limits_i8xx_lvds = {
 127         .dot = { .min = 25000, .max = 350000 },
 128         .vco = { .min = 930000, .max = 1400000 },
 129         .n = { .min = 3, .max = 16 },
 130         .m = { .min = 96, .max = 140 },
 131         .m1 = { .min = 18, .max = 26 },
 132         .m2 = { .min = 6, .max = 16 },
 133         .p = { .min = 4, .max = 128 },
 134         .p1 = { .min = 1, .max = 6 },
 135         .p2 = { .dot_limit = 165000,
 136                 .p2_slow = 14, .p2_fast = 7 },
 137         .find_pll = intel_find_best_PLL,
 138 };
 139
 140 static const intel_limit_t intel_limits_i9xx_sdvo = {
 141         .dot = { .min = 20000, .max = 400000 },
 142         .vco = { .min = 1400000, .max = 2800000 },
 143         .n = { .min = 1, .max = 6 },
 144         .m = { .min = 70, .max = 120 },
 145         .m1 = { .min = 10, .max = 22 },
 146         .m2 = { .min = 5, .max = 9 },
 147         .p = { .min = 5, .max = 80 },
 148         .p1 = { .min = 1, .max = 8 },
 149         .p2 = { .dot_limit = 200000,
 150                 .p2_slow = 10, .p2_fast = 5 },
 151         .find_pll = intel_find_best_PLL,
 152 };
 153
 154 static const intel_limit_t intel_limits_i9xx_lvds = {
 155         .dot = { .min = 20000, .max = 400000 },
 156         .vco = { .min = 1400000, .max = 2800000 },
 157         .n = { .min = 1, .max = 6 },
 158         .m = { .min = 70, .max = 120 },
 159         .m1 = { .min = 10, .max = 22 },
 160         .m2 = { .min = 5, .max = 9 },
 161         .p = { .min = 7, .max = 98 },
 162         .p1 = { .min = 1, .max = 8 },
 163         .p2 = { .dot_limit = 112000,
 164                 .p2_slow = 14, .p2_fast = 7 },
 165         .find_pll = intel_find_best_PLL,
 166 };
 167
 168
 169 static const intel_limit_t intel_limits_g4x_sdvo = {
 170         .dot = { .min = 25000, .max = 270000 },
 171         .vco = { .min = 1750000, .max = 3500000},
 172         .n = { .min = 1, .max = 4 },
 173         .m = { .min = 104, .max = 138 },
 174         .m1 = { .min = 17, .max = 23 },
 175         .m2 = { .min = 5, .max = 11 },
 176         .p = { .min = 10, .max = 30 },
 177         .p1 = { .min = 1, .max = 3},
 178         .p2 = { .dot_limit = 270000,
 179                 .p2_slow = 10,
 180                 .p2_fast = 10
 181         },
 182         .find_pll = intel_g4x_find_best_PLL,
 183 };
 184
 185 static const intel_limit_t intel_limits_g4x_hdmi = {
 186         .dot = { .min = 22000, .max = 400000 },
 187         .vco = { .min = 1750000, .max = 3500000},
 188         .n = { .min = 1, .max = 4 },
 189         .m = { .min = 104, .max = 138 },
 190         .m1 = { .min = 16, .max = 23 },
 191         .m2 = { .min = 5, .max = 11 },
 192         .p = { .min = 5, .max = 80 },
 193         .p1 = { .min = 1, .max = 8},
 194         .p2 = { .dot_limit = 165000,
 195                 .p2_slow = 10, .p2_fast = 5 },
 196         .find_pll = intel_g4x_find_best_PLL,
 197 };
 198
 199 static const intel_limit_t intel_limits_g4x_single_channel_lvds = {
 200         .dot = { .min = 20000, .max = 115000 },
 201         .vco = { .min = 1750000, .max = 3500000 },
 202         .n = { .min = 1, .max = 3 },
 203         .m = { .min = 104, .max = 138 },
 204         .m1 = { .min = 17, .max = 23 },
 205         .m2 = { .min = 5, .max = 11 },
 206         .p = { .min = 28, .max = 112 },
 207         .p1 = { .min = 2, .max = 8 },
 208         .p2 = { .dot_limit = 0,
 209                 .p2_slow = 14, .p2_fast = 14
 210         },
 211         .find_pll = intel_g4x_find_best_PLL,
 212 };
 213
 214 static const intel_limit_t intel_limits_g4x_dual_channel_lvds = {
 215         .dot = { .min = 80000, .max = 224000 },
 216         .vco = { .min = 1750000, .max = 3500000 },
 217         .n = { .min = 1, .max = 3 },
 218         .m = { .min = 104, .max = 138 },
 219         .m1 = { .min = 17, .max = 23 },
 220         .m2 = { .min = 5, .max = 11 },
 221         .p = { .min = 14, .max = 42 },
 222         .p1 = { .min = 2, .max = 6 },
 223         .p2 = { .dot_limit = 0,
 224                 .p2_slow = 7, .p2_fast = 7
 225         },
 226         .find_pll = intel_g4x_find_best_PLL,
 227 };
 228
 229 static const intel_limit_t intel_limits_g4x_display_port = {
 230         .dot = { .min = 161670, .max = 227000 },
 231         .vco = { .min = 1750000, .max = 3500000},
 232         .n = { .min = 1, .max = 2 },
 233         .m = { .min = 97, .max = 108 },
 234         .m1 = { .min = 0x10, .max = 0x12 },
 235         .m2 = { .min = 0x05, .max = 0x06 },
 236         .p = { .min = 10, .max = 20 },
 237         .p1 = { .min = 1, .max = 2},
 238         .p2 = { .dot_limit = 0,
 239                 .p2_slow = 10, .p2_fast = 10 },
 240         .find_pll = intel_find_pll_g4x_dp,
 241 };
 242
 243 static const intel_limit_t intel_limits_pineview_sdvo = {
 244         .dot = { .min = 20000, .max = 400000},
 245         .vco = { .min = 1700000, .max = 3500000 },
 246         /* Pineview's Ncounter is a ring counter */
 247         .n = { .min = 3, .max = 6 },
 248         .m = { .min = 2, .max = 256 },
 249         /* Pineview only has one combined m divider, which we treat as m2. */
 250         .m1 = { .min = 0, .max = 0 },
 251         .m2 = { .min = 0, .max = 254 },
 252         .p = { .min = 5, .max = 80 },
 253         .p1 = { .min = 1, .max = 8 },
 254         .p2 = { .dot_limit = 200000,
 255                 .p2_slow = 10, .p2_fast = 5 },
 256         .find_pll = intel_find_best_PLL,
 257 };
 258
 259 static const intel_limit_t intel_limits_pineview_lvds = {
 260         .dot = { .min = 20000, .max = 400000 },
 261         .vco = { .min = 1700000, .max = 3500000 },
 262         .n = { .min = 3, .max = 6 },
 263         .m = { .min = 2, .max = 256 },
 264         .m1 = { .min = 0, .max = 0 },
 265         .m2 = { .min = 0, .max = 254 },
 266         .p = { .min = 7, .max = 112 },
 267         .p1 = { .min = 1, .max = 8 },
 268         .p2 = { .dot_limit = 112000,
 269                 .p2_slow = 14, .p2_fast = 14 },
 270         .find_pll = intel_find_best_PLL,
 271 };
 272
 273 /* Ironlake / Sandybridge
 274  *
 275  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 276  * the range value for them is (actual_value - 2).
 277  */
 278 static const intel_limit_t intel_limits_ironlake_dac = {
 279         .dot = { .min = 25000, .max = 350000 },
 280         .vco = { .min = 1760000, .max = 3510000 },
 281         .n = { .min = 1, .max = 5 },
 282         .m = { .min = 79, .max = 127 },
 283         .m1 = { .min = 12, .max = 22 },
 284         .m2 = { .min = 5, .max = 9 },
 285         .p = { .min = 5, .max = 80 },
 286         .p1 = { .min = 1, .max = 8 },
 287         .p2 = { .dot_limit = 225000,
 288                 .p2_slow = 10, .p2_fast = 5 },
 289         .find_pll = intel_g4x_find_best_PLL,
 290 };
 291
 292 static const intel_limit_t intel_limits_ironlake_single_lvds = {
 293         .dot = { .min = 25000, .max = 350000 },
 294         .vco = { .min = 1760000, .max = 3510000 },
 295         .n = { .min = 1, .max = 3 },
 296         .m = { .min = 79, .max = 118 },
 297         .m1 = { .min = 12, .max = 22 },
 298         .m2 = { .min = 5, .max = 9 },
 299         .p = { .min = 28, .max = 112 },
 300         .p1 = { .min = 2, .max = 8 },
 301         .p2 = { .dot_limit = 225000,
 302                 .p2_slow = 14, .p2_fast = 14 },
 303         .find_pll = intel_g4x_find_best_PLL,
 304 };
 305
 306 static const intel_limit_t intel_limits_ironlake_dual_lvds = {
 307         .dot = { .min = 25000, .max = 350000 },
 308         .vco = { .min = 1760000, .max = 3510000 },
 309         .n = { .min = 1, .max = 3 },
 310         .m = { .min = 79, .max = 127 },
 311         .m1 = { .min = 12, .max = 22 },
 312         .m2 = { .min = 5, .max = 9 },
 313         .p = { .min = 14, .max = 56 },
 314         .p1 = { .min = 2, .max = 8 },
 315         .p2 = { .dot_limit = 225000,
 316                 .p2_slow = 7, .p2_fast = 7 },
 317         .find_pll = intel_g4x_find_best_PLL,
 318 };
 319
 320 /* LVDS 100mhz refclk limits. */
 321 static const intel_limit_t intel_limits_ironlake_single_lvds_100m = {
 322         .dot = { .min = 25000, .max = 350000 },
 323         .vco = { .min = 1760000, .max = 3510000 },
 324         .n = { .min = 1, .max = 2 },
 325         .m = { .min = 79, .max = 126 },
 326         .m1 = { .min = 12, .max = 22 },
 327         .m2 = { .min = 5, .max = 9 },
 328         .p = { .min = 28, .max = 112 },
 329         .p1 = { .min = 2, .max = 8 },
 330         .p2 = { .dot_limit = 225000,
 331                 .p2_slow = 14, .p2_fast = 14 },
 332         .find_pll = intel_g4x_find_best_PLL,
 333 };
 334
 335 static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = {
 336         .dot = { .min = 25000, .max = 350000 },
 337         .vco = { .min = 1760000, .max = 3510000 },
 338         .n = { .min = 1, .max = 3 },
 339         .m = { .min = 79, .max = 126 },
 340         .m1 = { .min = 12, .max = 22 },
 341         .m2 = { .min = 5, .max = 9 },
 342         .p = { .min = 14, .max = 42 },
 343         .p1 = { .min = 2, .max = 6 },
 344         .p2 = { .dot_limit = 225000,
 345                 .p2_slow = 7, .p2_fast = 7 },
 346         .find_pll = intel_g4x_find_best_PLL,
 347 };
 348
 349 static const intel_limit_t intel_limits_ironlake_display_port = {
 350         .dot = { .min = 25000, .max = 350000 },
 351         .vco = { .min = 1760000, .max = 3510000},
 352         .n = { .min = 1, .max = 2 },
 353         .m = { .min = 81, .max = 90 },
 354         .m1 = { .min = 12, .max = 22 },
 355         .m2 = { .min = 5, .max = 9 },
 356         .p = { .min = 10, .max = 20 },
 357         .p1 = { .min = 1, .max = 2},
 358         .p2 = { .dot_limit = 0,
 359                 .p2_slow = 10, .p2_fast = 10 },
 360         .find_pll = intel_find_pll_ironlake_dp,
 361 };
 362
 363 u32 intel_dpio_read(struct drm_i915_private *dev_priv, int reg)
 364 {
 365         unsigned long flags;
 366         u32 val = 0;
 367
 368         spin_lock_irqsave(&dev_priv->dpio_lock, flags);
 369         if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
 370                 DRM_ERROR("DPIO idle wait timed out\n");
 371                 goto out_unlock;
 372         }
 373
 374         I915_WRITE(DPIO_REG, reg);
 375         I915_WRITE(DPIO_PKT, DPIO_RID | DPIO_OP_READ | DPIO_PORTID |
 376                    DPIO_BYTE);
 377         if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
 378                 DRM_ERROR("DPIO read wait timed out\n");
 379                 goto out_unlock;
 380         }
 381         val = I915_READ(DPIO_DATA);
 382
 383 out_unlock:
 384         spin_unlock_irqrestore(&dev_priv->dpio_lock, flags);
 385         return val;
 386 }
 387
 388 static void intel_dpio_write(struct drm_i915_private *dev_priv, int reg,
 389                              u32 val)
 390 {
 391         unsigned long flags;
 392
 393         spin_lock_irqsave(&dev_priv->dpio_lock, flags);
 394         if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
 395                 DRM_ERROR("DPIO idle wait timed out\n");
 396                 goto out_unlock;
 397         }
 398
 399         I915_WRITE(DPIO_DATA, val);
 400         I915_WRITE(DPIO_REG, reg);
 401         I915_WRITE(DPIO_PKT, DPIO_RID | DPIO_OP_WRITE | DPIO_PORTID |
 402                    DPIO_BYTE);
 403         if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100))
 404                 DRM_ERROR("DPIO write wait timed out\n");
 405
 406 out_unlock:
 407         spin_unlock_irqrestore(&dev_priv->dpio_lock, flags);
 408 }
 409
 410 static void vlv_init_dpio(struct drm_device *dev)
 411 {
 412         struct drm_i915_private *dev_priv = dev->dev_private;
 413
 414         /* Reset the DPIO config */
 415         I915_WRITE(DPIO_CTL, 0);
 416         POSTING_READ(DPIO_CTL);
 417         I915_WRITE(DPIO_CTL, 1);
 418         POSTING_READ(DPIO_CTL);
 419 }
 420
 421 static bool is_dual_link_lvds(struct drm_i915_private *dev_priv,
 422                               unsigned int reg)
 423 {
 424         unsigned int val;
 425
 426         /* use the module option value if specified */
 427         if (i915_lvds_channel_mode > 0)
 428                 return i915_lvds_channel_mode == 2;
 429
 430         if (dev_priv->lvds_val)
 431                 val = dev_priv->lvds_val;
 432         else {
 433                 /* BIOS should set the proper LVDS register value at boot, but
 434                  * in reality, it doesn't set the value when the lid is closed;
 435                  * we need to check "the value to be set" in VBT when LVDS
 436                  * register is uninitialized.
 437                  */
 438                 val = I915_READ(reg);
 439                 if (!(val & ~LVDS_DETECTED))
 440                         val = dev_priv->bios_lvds_val;
 441                 dev_priv->lvds_val = val;
 442         }
 443         return (val & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP;
 444 }
 445
 446 static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc,
 447                                                 int refclk)
 448 {
 449         struct drm_device *dev = crtc->dev;
 450         struct drm_i915_private *dev_priv = dev->dev_private;
 451         const intel_limit_t *limit;
 452
 453         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
 454                 if (is_dual_link_lvds(dev_priv, PCH_LVDS)) {
 455                         /* LVDS dual channel */
 456                         if (refclk == 100000)
 457                                 limit = &intel_limits_ironlake_dual_lvds_100m;
 458                         else
 459                                 limit = &intel_limits_ironlake_dual_lvds;
 460                 } else {
 461                         if (refclk == 100000)
 462                                 limit = &intel_limits_ironlake_single_lvds_100m;
 463                         else
 464                                 limit = &intel_limits_ironlake_single_lvds;
 465                 }
 466         } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
 467                         HAS_eDP)
 468                 limit = &intel_limits_ironlake_display_port;
 469         else
 470                 limit = &intel_limits_ironlake_dac;
 471
 472         return limit;
 473 }
 474
 475 static const intel_limit_t *intel_g4x_limit(struct drm_crtc *crtc)
 476 {
 477         struct drm_device *dev = crtc->dev;
 478         struct drm_i915_private *dev_priv = dev->dev_private;
 479         const intel_limit_t *limit;
 480
 481         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
 482                 if (is_dual_link_lvds(dev_priv, LVDS))
 483                         /* LVDS with dual channel */
 484                         limit = &intel_limits_g4x_dual_channel_lvds;
 485                 else
 486                         /* LVDS with dual channel */
 487                         limit = &intel_limits_g4x_single_channel_lvds;
 488         } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI) ||
 489                    intel_pipe_has_type(crtc, INTEL_OUTPUT_ANALOG)) {
 490                 limit = &intel_limits_g4x_hdmi;
 491         } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO)) {
 492                 limit = &intel_limits_g4x_sdvo;
 493         } else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
 494                 limit = &intel_limits_g4x_display_port;
 495         } else /* The option is for other outputs */
 496                 limit = &intel_limits_i9xx_sdvo;
 497
 498         return limit;
 499 }
 500
 501 static const intel_limit_t *intel_limit(struct drm_crtc *crtc, int refclk)
 502 {
 503         struct drm_device *dev = crtc->dev;
 504         const intel_limit_t *limit;
 505
 506         if (HAS_PCH_SPLIT(dev))
 507                 limit = intel_ironlake_limit(crtc, refclk);
 508         else if (IS_G4X(dev)) {
 509                 limit = intel_g4x_limit(crtc);
 510         } else if (IS_PINEVIEW(dev)) {
 511                 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
 512                         limit = &intel_limits_pineview_lvds;
 513                 else
 514                         limit = &intel_limits_pineview_sdvo;
 515         } else if (!IS_GEN2(dev)) {
 516                 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
 517                         limit = &intel_limits_i9xx_lvds;
 518                 else
 519                         limit = &intel_limits_i9xx_sdvo;
 520         } else {
 521                 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
 522                         limit = &intel_limits_i8xx_lvds;
 523                 else
 524                         limit = &intel_limits_i8xx_dvo;
 525         }
 526         return limit;
 527 }
 528
 529 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 530 static void pineview_clock(int refclk, intel_clock_t *clock)
 531 {
 532         clock->m = clock->m2 + 2;
 533         clock->p = clock->p1 * clock->p2;
 534         clock->vco = refclk * clock->m / clock->n;
 535         clock->dot = clock->vco / clock->p;
 536 }
 537
 538 static void intel_clock(struct drm_device *dev, int refclk, intel_clock_t *clock)
 539 {
 540         if (IS_PINEVIEW(dev)) {
 541                 pineview_clock(refclk, clock);
 542                 return;
 543         }
 544         clock->m = 5 * (clock->m1 + 2) + (clock->m2 + 2);
 545         clock->p = clock->p1 * clock->p2;
 546         clock->vco = refclk * clock->m / (clock->n + 2);
 547         clock->dot = clock->vco / clock->p;
 548 }
 549
 550 /**
 551  * Returns whether any output on the specified pipe is of the specified type
 552  */
 553 bool intel_pipe_has_type(struct drm_crtc *crtc, int type)
 554 {
 555         struct drm_device *dev = crtc->dev;
 556         struct drm_mode_config *mode_config = &dev->mode_config;
 557         struct intel_encoder *encoder;
 558
 559         list_for_each_entry(encoder, &mode_config->encoder_list, base.head)
 560                 if (encoder->base.crtc == crtc && encoder->type == type)
 561                         return true;
 562
 563         return false;
 564 }
 565
 566 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 567 /**
 568  * Returns whether the given set of divisors are valid for a given refclk with
 569  * the given connectors.
 570  */
 571
 572 static bool intel_PLL_is_valid(struct drm_device *dev,
 573                                const intel_limit_t *limit,
 574                                const intel_clock_t *clock)
 575 {
 576         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 577                 INTELPllInvalid("p1 out of range\n");
 578         if (clock->p   < limit->p.min   || limit->p.max   < clock->p)
 579                 INTELPllInvalid("p out of range\n");
 580         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 581                 INTELPllInvalid("m2 out of range\n");
 582         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 583                 INTELPllInvalid("m1 out of range\n");
 584         if (clock->m1 <= clock->m2 && !IS_PINEVIEW(dev))
 585                 INTELPllInvalid("m1 <= m2\n");
 586         if (clock->m   < limit->m.min   || limit->m.max   < clock->m)
 587                 INTELPllInvalid("m out of range\n");
 588         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 589                 INTELPllInvalid("n out of range\n");
 590         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 591                 INTELPllInvalid("vco out of range\n");
 592         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 593          * connector, etc., rather than just a single range.
 594          */
 595         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 596                 INTELPllInvalid("dot out of range\n");
 597
 598         return true;
 599 }
 600
 601 static bool
 602 intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
 603                     int target, int refclk, intel_clock_t *match_clock,
 604                     intel_clock_t *best_clock)
 605
 606 {
 607         struct drm_device *dev = crtc->dev;
 608         struct drm_i915_private *dev_priv = dev->dev_private;
 609         intel_clock_t clock;
 610         int err = target;
 611
 612         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
 613             (I915_READ(LVDS)) != 0) {
 614                 /*
 615                  * For LVDS, if the panel is on, just rely on its current
 616                  * settings for dual-channel.  We haven't figured out how to
 617                  * reliably set up different single/dual channel state, if we
 618                  * even can.
 619                  */
 620                 if (is_dual_link_lvds(dev_priv, LVDS))
 621                         clock.p2 = limit->p2.p2_fast;
 622                 else
 623                         clock.p2 = limit->p2.p2_slow;
 624         } else {
 625                 if (target < limit->p2.dot_limit)
 626                         clock.p2 = limit->p2.p2_slow;
 627                 else
 628                         clock.p2 = limit->p2.p2_fast;
 629         }
 630
 631         memset(best_clock, 0, sizeof(*best_clock));
 632
 633         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 634              clock.m1++) {
 635                 for (clock.m2 = limit->m2.min;
 636                      clock.m2 <= limit->m2.max; clock.m2++) {
 637                         /* m1 is always 0 in Pineview */
 638                         if (clock.m2 >= clock.m1 && !IS_PINEVIEW(dev))
 639                                 break;
 640                         for (clock.n = limit->n.min;
 641                              clock.n <= limit->n.max; clock.n++) {
 642                                 for (clock.p1 = limit->p1.min;
 643                                         clock.p1 <= limit->p1.max; clock.p1++) {
 644                                         int this_err;
 645
 646                                         intel_clock(dev, refclk, &clock);
 647                                         if (!intel_PLL_is_valid(dev, limit,
 648                                                                 &clock))
 649                                                 continue;
 650                                         if (match_clock &&
 651                                             clock.p != match_clock->p)
 652                                                 continue;
 653
 654                                         this_err = abs(clock.dot - target);
 655                                         if (this_err < err) {
 656                                                 *best_clock = clock;
 657                                                 err = this_err;
 658                                         }
 659                                 }
 660                         }
 661                 }
 662         }
 663
 664         return (err != target);
 665 }
 666
 667 static bool
 668 intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
 669                         int target, int refclk, intel_clock_t *match_clock,
 670                         intel_clock_t *best_clock)
 671 {
 672         struct drm_device *dev = crtc->dev;
 673         struct drm_i915_private *dev_priv = dev->dev_private;
 674         intel_clock_t clock;
 675         int max_n;
 676         bool found;
 677         /* approximately equals target * 0.00585 */
 678         int err_most = (target >> 8) + (target >> 9);
 679         found = false;
 680
 681         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
 682                 int lvds_reg;
 683
 684                 if (HAS_PCH_SPLIT(dev))
 685                         lvds_reg = PCH_LVDS;
 686                 else
 687                         lvds_reg = LVDS;
 688                 if ((I915_READ(lvds_reg) & LVDS_CLKB_POWER_MASK) ==
 689                     LVDS_CLKB_POWER_UP)
 690                         clock.p2 = limit->p2.p2_fast;
 691                 else
 692                         clock.p2 = limit->p2.p2_slow;
 693         } else {
 694                 if (target < limit->p2.dot_limit)
 695                         clock.p2 = limit->p2.p2_slow;
 696                 else
 697                         clock.p2 = limit->p2.p2_fast;
 698         }
 699
 700         memset(best_clock, 0, sizeof(*best_clock));
 701         max_n = limit->n.max;
 702         /* based on hardware requirement, prefer smaller n to precision */
 703         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 704                 /* based on hardware requirement, prefere larger m1,m2 */
 705                 for (clock.m1 = limit->m1.max;
 706                      clock.m1 >= limit->m1.min; clock.m1--) {
 707                         for (clock.m2 = limit->m2.max;
 708                              clock.m2 >= limit->m2.min; clock.m2--) {
 709                                 for (clock.p1 = limit->p1.max;
 710                                      clock.p1 >= limit->p1.min; clock.p1--) {
 711                                         int this_err;
 712
 713                                         intel_clock(dev, refclk, &clock);
 714                                         if (!intel_PLL_is_valid(dev, limit,
 715                                                                 &clock))
 716                                                 continue;
 717                                         if (match_clock &&
 718                                             clock.p != match_clock->p)
 719                                                 continue;
 720
 721                                         this_err = abs(clock.dot - target);
 722                                         if (this_err < err_most) {
 723                                                 *best_clock = clock;
 724                                                 err_most = this_err;
 725                                                 max_n = clock.n;
 726                                                 found = true;
 727                                         }
 728                                 }
 729                         }
 730                 }
 731         }
 732         return found;
 733 }
 734
 735 static bool
 736 intel_find_pll_ironlake_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
 737                            int target, int refclk, intel_clock_t *match_clock,
 738                            intel_clock_t *best_clock)
 739 {
 740         struct drm_device *dev = crtc->dev;
 741         intel_clock_t clock;
 742
 743         if (target < 200000) {
 744                 clock.n = 1;
 745                 clock.p1 = 2;
 746                 clock.p2 = 10;
 747                 clock.m1 = 12;
 748                 clock.m2 = 9;
 749         } else {
 750                 clock.n = 2;
 751                 clock.p1 = 1;
 752                 clock.p2 = 10;
 753                 clock.m1 = 14;
 754                 clock.m2 = 8;
 755         }
 756         intel_clock(dev, refclk, &clock);
 757         memcpy(best_clock, &clock, sizeof(intel_clock_t));
 758         return true;
 759 }
 760
 761 /* DisplayPort has only two frequencies, 162MHz and 270MHz */
 762 static bool
 763 intel_find_pll_g4x_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
 764                       int target, int refclk, intel_clock_t *match_clock,
 765                       intel_clock_t *best_clock)
 766 {
 767         intel_clock_t clock;
 768         if (target < 200000) {
 769                 clock.p1 = 2;
 770                 clock.p2 = 10;
 771                 clock.n = 2;
 772                 clock.m1 = 23;
 773                 clock.m2 = 8;
 774         } else {
 775                 clock.p1 = 1;
 776                 clock.p2 = 10;
 777                 clock.n = 1;
 778                 clock.m1 = 14;
 779                 clock.m2 = 2;
 780         }
 781         clock.m = 5 * (clock.m1 + 2) + (clock.m2 + 2);
 782         clock.p = (clock.p1 * clock.p2);
 783         clock.dot = 96000 * clock.m / (clock.n + 2) / clock.p;
 784         clock.vco = 0;
 785         memcpy(best_clock, &clock, sizeof(intel_clock_t));
 786         return true;
 787 }
 788
 789 /**
 790  * intel_wait_for_vblank - wait for vblank on a given pipe
 791  * @dev: drm device
 792  * @pipe: pipe to wait for
 793  *
 794  * Wait for vblank to occur on a given pipe.  Needed for various bits of
 795  * mode setting code.
 796  */
 797 void intel_wait_for_vblank(struct drm_device *dev, int pipe)
 798 {
 799         struct drm_i915_private *dev_priv = dev->dev_private;
 800         int pipestat_reg = PIPESTAT(pipe);
 801
 802         /* Clear existing vblank status. Note this will clear any other
 803          * sticky status fields as well.
 804          *
 805          * This races with i915_driver_irq_handler() with the result
 806          * that either function could miss a vblank event.  Here it is not
 807          * fatal, as we will either wait upon the next vblank interrupt or
 808          * timeout.  Generally speaking intel_wait_for_vblank() is only
 809          * called during modeset at which time the GPU should be idle and
 810          * should *not* be performing page flips and thus not waiting on
 811          * vblanks...
 812          * Currently, the result of us stealing a vblank from the irq
 813          * handler is that a single frame will be skipped during swapbuffers.
 814          */
 815         I915_WRITE(pipestat_reg,
 816                    I915_READ(pipestat_reg) | PIPE_VBLANK_INTERRUPT_STATUS);
 817
 818         /* Wait for vblank interrupt bit to set */
 819         if (wait_for(I915_READ(pipestat_reg) &
 820                      PIPE_VBLANK_INTERRUPT_STATUS,
 821                      50))
 822                 DRM_DEBUG_KMS("vblank wait timed out\n");
 823 }
 824
 825 /*
 826  * intel_wait_for_pipe_off - wait for pipe to turn off
 827  * @dev: drm device
 828  * @pipe: pipe to wait for
 829  *
 830  * After disabling a pipe, we can't wait for vblank in the usual way,
 831  * spinning on the vblank interrupt status bit, since we won't actually
 832  * see an interrupt when the pipe is disabled.
 833  *
 834  * On Gen4 and above:
 835  *   wait for the pipe register state bit to turn off
 836  *
 837  * Otherwise:
 838  *   wait for the display line value to settle (it usually
 839  *   ends up stopping at the start of the next frame).
 840  *
 841  */
 842 void intel_wait_for_pipe_off(struct drm_device *dev, int pipe)
 843 {
 844         struct drm_i915_private *dev_priv = dev->dev_private;
 845
 846         if (INTEL_INFO(dev)->gen >= 4) {
 847                 int reg = PIPECONF(pipe);
 848
 849                 /* Wait for the Pipe State to go off */
 850                 if (wait_for((I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0,
 851                              100))
 852                         DRM_DEBUG_KMS("pipe_off wait timed out\n");
 853         } else {
 854                 u32 last_line;
 855                 int reg = PIPEDSL(pipe);
 856                 unsigned long timeout = jiffies + msecs_to_jiffies(100);
 857
 858                 /* Wait for the display line to settle */
 859                 do {
 860                         last_line = I915_READ(reg) & DSL_LINEMASK;
 861                         mdelay(5);
 862                 } while (((I915_READ(reg) & DSL_LINEMASK) != last_line) &&
 863                          time_after(timeout, jiffies));
 864                 if (time_after(jiffies, timeout))
 865                         DRM_DEBUG_KMS("pipe_off wait timed out\n");
 866         }
 867 }
 868
 869 static const char *state_string(bool enabled)
 870 {
 871         return enabled ? "on" : "off";
 872 }
 873
 874 /* Only for pre-ILK configs */
 875 static void assert_pll(struct drm_i915_private *dev_priv,
 876                        enum pipe pipe, bool state)
 877 {
 878         int reg;
 879         u32 val;
 880         bool cur_state;
 881
 882         reg = DPLL(pipe);
 883         val = I915_READ(reg);
 884         cur_state = !!(val & DPLL_VCO_ENABLE);
 885         WARN(cur_state != state,
 886              "PLL state assertion failure (expected %s, current %s)\n",
 887              state_string(state), state_string(cur_state));
 888 }
 889 #define assert_pll_enabled(d, p) assert_pll(d, p, true)
 890 #define assert_pll_disabled(d, p) assert_pll(d, p, false)
 891
 892 /* For ILK+ */
 893 static void assert_pch_pll(struct drm_i915_private *dev_priv,
 894                            enum pipe pipe, bool state)
 895 {
 896         int reg;
 897         u32 val;
 898         bool cur_state;
 899
 900         if (HAS_PCH_CPT(dev_priv->dev)) {
 901                 u32 pch_dpll;
 902
 903                 pch_dpll = I915_READ(PCH_DPLL_SEL);
 904
 905                 /* Make sure the selected PLL is enabled to the transcoder */
 906                 WARN(!((pch_dpll >> (4 * pipe)) & 8),
 907                      "transcoder %d PLL not enabled\n", pipe);
 908
 909                 /* Convert the transcoder pipe number to a pll pipe number */
 910                 pipe = (pch_dpll >> (4 * pipe)) & 1;
 911         }
 912
 913         reg = PCH_DPLL(pipe);
 914         val = I915_READ(reg);
 915         cur_state = !!(val & DPLL_VCO_ENABLE);
 916         WARN(cur_state != state,
 917              "PCH PLL state assertion failure (expected %s, current %s)\n",
 918              state_string(state), state_string(cur_state));
 919 }
 920 #define assert_pch_pll_enabled(d, p) assert_pch_pll(d, p, true)
 921 #define assert_pch_pll_disabled(d, p) assert_pch_pll(d, p, false)
 922
 923 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
 924                           enum pipe pipe, bool state)
 925 {
 926         int reg;
 927         u32 val;
 928         bool cur_state;
 929
 930         reg = FDI_TX_CTL(pipe);
 931         val = I915_READ(reg);
 932         cur_state = !!(val & FDI_TX_ENABLE);
 933         WARN(cur_state != state,
 934              "FDI TX state assertion failure (expected %s, current %s)\n",
 935              state_string(state), state_string(cur_state));
 936 }
 937 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
 938 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
 939
 940 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
 941                           enum pipe pipe, bool state)
 942 {
 943         int reg;
 944         u32 val;
 945         bool cur_state;
 946
 947         reg = FDI_RX_CTL(pipe);
 948         val = I915_READ(reg);
 949         cur_state = !!(val & FDI_RX_ENABLE);
 950         WARN(cur_state != state,
 951              "FDI RX state assertion failure (expected %s, current %s)\n",
 952              state_string(state), state_string(cur_state));
 953 }
 954 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
 955 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
 956
 957 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
 958                                       enum pipe pipe)
 959 {
 960         int reg;
 961         u32 val;
 962
 963         /* ILK FDI PLL is always enabled */
 964         if (dev_priv->info->gen == 5)
 965                 return;
 966
 967         reg = FDI_TX_CTL(pipe);
 968         val = I915_READ(reg);
 969         WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
 970 }
 971
 972 static void assert_fdi_rx_pll_enabled(struct drm_i915_private *dev_priv,
 973                                       enum pipe pipe)
 974 {
 975         int reg;
 976         u32 val;
 977
 978         reg = FDI_RX_CTL(pipe);
 979         val = I915_READ(reg);
 980         WARN(!(val & FDI_RX_PLL_ENABLE), "FDI RX PLL assertion failure, should be active but is disabled\n");
 981 }
 982
 983 static void assert_panel_unlocked(struct drm_i915_private *dev_priv,
 984                                   enum pipe pipe)
 985 {
 986         int pp_reg, lvds_reg;
 987         u32 val;
 988         enum pipe panel_pipe = PIPE_A;
 989         bool locked = true;
 990
 991         if (HAS_PCH_SPLIT(dev_priv->dev)) {
 992                 pp_reg = PCH_PP_CONTROL;
 993                 lvds_reg = PCH_LVDS;
 994         } else {
 995                 pp_reg = PP_CONTROL;
 996                 lvds_reg = LVDS;
 997         }
 998
 999         val = I915_READ(pp_reg);
1000         if (!(val & PANEL_POWER_ON) ||
1001             ((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS))
1002                 locked = false;
1003
1004         if (I915_READ(lvds_reg) & LVDS_PIPEB_SELECT)
1005                 panel_pipe = PIPE_B;
1006
1007         WARN(panel_pipe == pipe && locked,
1008              "panel assertion failure, pipe %c regs locked\n",
1009              pipe_name(pipe));
1010 }
1011
1012 void assert_pipe(struct drm_i915_private *dev_priv,
1013                  enum pipe pipe, bool state)
1014 {
1015         int reg;
1016         u32 val;
1017         bool cur_state;
1018
1019         /* if we need the pipe A quirk it must be always on */
1020         if (pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE)
1021                 state = true;
1022
1023         reg = PIPECONF(pipe);
1024         val = I915_READ(reg);
1025         cur_state = !!(val & PIPECONF_ENABLE);
1026         WARN(cur_state != state,
1027              "pipe %c assertion failure (expected %s, current %s)\n",
1028              pipe_name(pipe), state_string(state), state_string(cur_state));
1029 }
1030
1031 static void assert_plane(struct drm_i915_private *dev_priv,
1032                          enum plane plane, bool state)
1033 {
1034         int reg;
1035         u32 val;
1036         bool cur_state;
1037
1038         reg = DSPCNTR(plane);
1039         val = I915_READ(reg);
1040         cur_state = !!(val & DISPLAY_PLANE_ENABLE);
1041         WARN(cur_state != state,
1042              "plane %c assertion failure (expected %s, current %s)\n",
1043              plane_name(plane), state_string(state), state_string(cur_state));
1044 }
1045
1046 #define assert_plane_enabled(d, p) assert_plane(d, p, true)
1047 #define assert_plane_disabled(d, p) assert_plane(d, p, false)
1048
1049 static void assert_planes_disabled(struct drm_i915_private *dev_priv,
1050                                    enum pipe pipe)
1051 {
1052         int reg, i;
1053         u32 val;
1054         int cur_pipe;
1055
1056         /* Planes are fixed to pipes on ILK+ */
1057         if (HAS_PCH_SPLIT(dev_priv->dev)) {
1058                 reg = DSPCNTR(pipe);
1059                 val = I915_READ(reg);
1060                 WARN((val & DISPLAY_PLANE_ENABLE),
1061                      "plane %c assertion failure, should be disabled but not\n",
1062                      plane_name(pipe));
1063                 return;
1064         }
1065
1066         /* Need to check both planes against the pipe */
1067         for (i = 0; i < 2; i++) {
1068                 reg = DSPCNTR(i);
1069                 val = I915_READ(reg);
1070                 cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
1071                         DISPPLANE_SEL_PIPE_SHIFT;
1072                 WARN((val & DISPLAY_PLANE_ENABLE) && pipe == cur_pipe,
1073                      "plane %c assertion failure, should be off on pipe %c but is still active\n",
1074                      plane_name(i), pipe_name(pipe));
1075         }
1076 }
1077
1078 static void assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
1079 {
1080         u32 val;
1081         bool enabled;
1082
1083         val = I915_READ(PCH_DREF_CONTROL);
1084         enabled = !!(val & (DREF_SSC_SOURCE_MASK | DREF_NONSPREAD_SOURCE_MASK |
1085                             DREF_SUPERSPREAD_SOURCE_MASK));
1086         WARN(!enabled, "PCH refclk assertion failure, should be active but is disabled\n");
1087 }
1088
1089 static void assert_transcoder_disabled(struct drm_i915_private *dev_priv,
1090                                        enum pipe pipe)
1091 {
1092         int reg;
1093         u32 val;
1094         bool enabled;
1095
1096         reg = TRANSCONF(pipe);
1097         val = I915_READ(reg);
1098         enabled = !!(val & TRANS_ENABLE);
1099         WARN(enabled,
1100              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1101              pipe_name(pipe));
1102 }
1103
1104 static bool dp_pipe_enabled(struct drm_i915_private *dev_priv,
1105                             enum pipe pipe, u32 port_sel, u32 val)
1106 {
1107         if ((val & DP_PORT_EN) == 0)
1108                 return false;
1109
1110         if (HAS_PCH_CPT(dev_priv->dev)) {
1111                 u32     trans_dp_ctl_reg = TRANS_DP_CTL(pipe);
1112                 u32     trans_dp_ctl = I915_READ(trans_dp_ctl_reg);
1113                 if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel)
1114                         return false;
1115         } else {
1116                 if ((val & DP_PIPE_MASK) != (pipe << 30))
1117                         return false;
1118         }
1119         return true;
1120 }
1121
1122 static bool hdmi_pipe_enabled(struct drm_i915_private *dev_priv,
1123                               enum pipe pipe, u32 val)
1124 {
1125         if ((val & PORT_ENABLE) == 0)
1126                 return false;
1127
1128         if (HAS_PCH_CPT(dev_priv->dev)) {
1129                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1130                         return false;
1131         } else {
1132                 if ((val & TRANSCODER_MASK) != TRANSCODER(pipe))
1133                         return false;
1134         }
1135         return true;
1136 }
1137
1138 static bool lvds_pipe_enabled(struct drm_i915_private *dev_priv,
1139                               enum pipe pipe, u32 val)
1140 {
1141         if ((val & LVDS_PORT_EN) == 0)
1142                 return false;
1143
1144         if (HAS_PCH_CPT(dev_priv->dev)) {
1145                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1146                         return false;
1147         } else {
1148                 if ((val & LVDS_PIPE_MASK) != LVDS_PIPE(pipe))
1149                         return false;
1150         }
1151         return true;
1152 }
1153
1154 static bool adpa_pipe_enabled(struct drm_i915_private *dev_priv,
1155                               enum pipe pipe, u32 val)
1156 {
1157         if ((val & ADPA_DAC_ENABLE) == 0)
1158                 return false;
1159         if (HAS_PCH_CPT(dev_priv->dev)) {
1160                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1161                         return false;
1162         } else {
1163                 if ((val & ADPA_PIPE_SELECT_MASK) != ADPA_PIPE_SELECT(pipe))
1164                         return false;
1165         }
1166         return true;
1167 }
1168
1169 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1170                                    enum pipe pipe, int reg, u32 port_sel)
1171 {
1172         u32 val = I915_READ(reg);
1173         WARN(dp_pipe_enabled(dev_priv, pipe, port_sel, val),
1174              "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
1175              reg, pipe_name(pipe));
1176 }
1177
1178 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1179                                      enum pipe pipe, int reg)
1180 {
1181         u32 val = I915_READ(reg);
1182         WARN(hdmi_pipe_enabled(dev_priv, val, pipe),
1183              "PCH HDMI (0x%08x) enabled on transcoder %c, should be disabled\n",
1184              reg, pipe_name(pipe));
1185 }
1186
1187 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1188                                       enum pipe pipe)
1189 {
1190         int reg;
1191         u32 val;
1192
1193         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
1194         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
1195         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
1196
1197         reg = PCH_ADPA;
1198         val = I915_READ(reg);
1199         WARN(adpa_pipe_enabled(dev_priv, val, pipe),
1200              "PCH VGA enabled on transcoder %c, should be disabled\n",
1201              pipe_name(pipe));
1202
1203         reg = PCH_LVDS;
1204         val = I915_READ(reg);
1205         WARN(lvds_pipe_enabled(dev_priv, val, pipe),
1206              "PCH LVDS enabled on transcoder %c, should be disabled\n",
1207              pipe_name(pipe));
1208
1209         assert_pch_hdmi_disabled(dev_priv, pipe, HDMIB);
1210         assert_pch_hdmi_disabled(dev_priv, pipe, HDMIC);
1211         assert_pch_hdmi_disabled(dev_priv, pipe, HDMID);
1212 }
1213
1214 /**
1215  * intel_enable_pll - enable a PLL
1216  * @dev_priv: i915 private structure
1217  * @pipe: pipe PLL to enable
1218  *
1219  * Enable @pipe's PLL so we can start pumping pixels from a plane.  Check to
1220  * make sure the PLL reg is writable first though, since the panel write
1221  * protect mechanism may be enabled.
1222  *
1223  * Note!  This is for pre-ILK only.
1224  */
1225 static void intel_enable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1226 {
1227         int reg;
1228         u32 val;
1229
1230         /* No really, not for ILK+ */
1231         BUG_ON(dev_priv->info->gen >= 5);
1232
1233         /* PLL is protected by panel, make sure we can write it */
1234         if (IS_MOBILE(dev_priv->dev) && !IS_I830(dev_priv->dev))
1235                 assert_panel_unlocked(dev_priv, pipe);
1236
1237         reg = DPLL(pipe);
1238         val = I915_READ(reg);
1239         val |= DPLL_VCO_ENABLE;
1240
1241         /* We do this three times for luck */
1242         I915_WRITE(reg, val);
1243         POSTING_READ(reg);
1244         udelay(150); /* wait for warmup */
1245         I915_WRITE(reg, val);
1246         POSTING_READ(reg);
1247         udelay(150); /* wait for warmup */
1248         I915_WRITE(reg, val);
1249         POSTING_READ(reg);
1250         udelay(150); /* wait for warmup */
1251 }
1252
1253 /**
1254  * intel_disable_pll - disable a PLL
1255  * @dev_priv: i915 private structure
1256  * @pipe: pipe PLL to disable
1257  *
1258  * Disable the PLL for @pipe, making sure the pipe is off first.
1259  *
1260  * Note!  This is for pre-ILK only.
1261  */
1262 static void intel_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1263 {
1264         int reg;
1265         u32 val;
1266
1267         /* Don't disable pipe A or pipe A PLLs if needed */
1268         if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
1269                 return;
1270
1271         /* Make sure the pipe isn't still relying on us */
1272         assert_pipe_disabled(dev_priv, pipe);
1273
1274         reg = DPLL(pipe);
1275         val = I915_READ(reg);
1276         val &= ~DPLL_VCO_ENABLE;
1277         I915_WRITE(reg, val);
1278         POSTING_READ(reg);
1279 }
1280
1281 /**
1282  * intel_enable_pch_pll - enable PCH PLL
1283  * @dev_priv: i915 private structure
1284  * @pipe: pipe PLL to enable
1285  *
1286  * The PCH PLL needs to be enabled before the PCH transcoder, since it
1287  * drives the transcoder clock.
1288  */
1289 static void intel_enable_pch_pll(struct drm_i915_private *dev_priv,
1290                                  enum pipe pipe)
1291 {
1292         int reg;
1293         u32 val;
1294
1295         if (pipe > 1)
1296                 return;
1297
1298         /* PCH only available on ILK+ */
1299         BUG_ON(dev_priv->info->gen < 5);
1300
1301         /* PCH refclock must be enabled first */
1302         assert_pch_refclk_enabled(dev_priv);
1303
1304         reg = PCH_DPLL(pipe);
1305         val = I915_READ(reg);
1306         val |= DPLL_VCO_ENABLE;
1307         I915_WRITE(reg, val);
1308         POSTING_READ(reg);
1309         udelay(200);
1310 }
1311
1312 static void intel_disable_pch_pll(struct drm_i915_private *dev_priv,
1313                                   enum pipe pipe)
1314 {
1315         int reg;
1316         u32 val, pll_mask = TRANSC_DPLL_ENABLE | TRANSC_DPLLB_SEL,
1317                 pll_sel = TRANSC_DPLL_ENABLE;
1318
1319         if (pipe > 1)
1320                 return;
1321
1322         /* PCH only available on ILK+ */
1323         BUG_ON(dev_priv->info->gen < 5);
1324
1325         /* Make sure transcoder isn't still depending on us */
1326         assert_transcoder_disabled(dev_priv, pipe);
1327
1328         if (pipe == 0)
1329                 pll_sel |= TRANSC_DPLLA_SEL;
1330         else if (pipe == 1)
1331                 pll_sel |= TRANSC_DPLLB_SEL;
1332
1333
1334         if ((I915_READ(PCH_DPLL_SEL) & pll_mask) == pll_sel)
1335                 return;
1336
1337         reg = PCH_DPLL(pipe);
1338         val = I915_READ(reg);
1339         val &= ~DPLL_VCO_ENABLE;
1340         I915_WRITE(reg, val);
1341         POSTING_READ(reg);
1342         udelay(200);
1343 }
1344
1345 static void intel_enable_transcoder(struct drm_i915_private *dev_priv,
1346                                     enum pipe pipe)
1347 {
1348         int reg;
1349         u32 val, pipeconf_val;
1350         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1351
1352         /* PCH only available on ILK+ */
1353         BUG_ON(dev_priv->info->gen < 5);
1354
1355         /* Make sure PCH DPLL is enabled */
1356         assert_pch_pll_enabled(dev_priv, pipe);
1357
1358         /* FDI must be feeding us bits for PCH ports */
1359         assert_fdi_tx_enabled(dev_priv, pipe);
1360         assert_fdi_rx_enabled(dev_priv, pipe);
1361
1362         reg = TRANSCONF(pipe);
1363         val = I915_READ(reg);
1364         pipeconf_val = I915_READ(PIPECONF(pipe));
1365
1366         if (HAS_PCH_IBX(dev_priv->dev)) {
1367                 /*
1368                  * make the BPC in transcoder be consistent with
1369                  * that in pipeconf reg.
1370                  */
1371                 val &= ~PIPE_BPC_MASK;
1372                 val |= pipeconf_val & PIPE_BPC_MASK;
1373         }
1374
1375         val &= ~TRANS_INTERLACE_MASK;
1376         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
1377                 if (HAS_PCH_IBX(dev_priv->dev) &&
1378                     intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO))
1379                         val |= TRANS_LEGACY_INTERLACED_ILK;
1380                 else
1381                         val |= TRANS_INTERLACED;
1382         else
1383                 val |= TRANS_PROGRESSIVE;
1384
1385         I915_WRITE(reg, val | TRANS_ENABLE);
1386         if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100))
1387                 DRM_ERROR("failed to enable transcoder %d\n", pipe);
1388 }
1389
1390 static void intel_disable_transcoder(struct drm_i915_private *dev_priv,
1391                                      enum pipe pipe)
1392 {
1393         int reg;
1394         u32 val;
1395
1396         /* FDI relies on the transcoder */
1397         assert_fdi_tx_disabled(dev_priv, pipe);
1398         assert_fdi_rx_disabled(dev_priv, pipe);
1399
1400         /* Ports must be off as well */
1401         assert_pch_ports_disabled(dev_priv, pipe);
1402
1403         reg = TRANSCONF(pipe);
1404         val = I915_READ(reg);
1405         val &= ~TRANS_ENABLE;
1406         I915_WRITE(reg, val);
1407         /* wait for PCH transcoder off, transcoder state */
1408         if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50))
1409                 DRM_ERROR("failed to disable transcoder %d\n", pipe);
1410 }
1411
1412 /**
1413  * intel_enable_pipe - enable a pipe, asserting requirements
1414  * @dev_priv: i915 private structure
1415  * @pipe: pipe to enable
1416  * @pch_port: on ILK+, is this pipe driving a PCH port or not
1417  *
1418  * Enable @pipe, making sure that various hardware specific requirements
1419  * are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc.
1420  *
1421  * @pipe should be %PIPE_A or %PIPE_B.
1422  *
1423  * Will wait until the pipe is actually running (i.e. first vblank) before
1424  * returning.
1425  */
1426 static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe,
1427                               bool pch_port)
1428 {
1429         int reg;
1430         u32 val;
1431
1432         /*
1433          * A pipe without a PLL won't actually be able to drive bits from
1434          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1435          * need the check.
1436          */
1437         if (!HAS_PCH_SPLIT(dev_priv->dev))
1438                 assert_pll_enabled(dev_priv, pipe);
1439         else {
1440                 if (pch_port) {
1441                         /* if driving the PCH, we need FDI enabled */
1442                         assert_fdi_rx_pll_enabled(dev_priv, pipe);
1443                         assert_fdi_tx_pll_enabled(dev_priv, pipe);
1444                 }
1445                 /* FIXME: assert CPU port conditions for SNB+ */
1446         }
1447
1448         reg = PIPECONF(pipe);
1449         val = I915_READ(reg);
1450         if (val & PIPECONF_ENABLE)
1451                 return;
1452
1453         I915_WRITE(reg, val | PIPECONF_ENABLE);
1454         intel_wait_for_vblank(dev_priv->dev, pipe);
1455 }
1456
1457 /**
1458  * intel_disable_pipe - disable a pipe, asserting requirements
1459  * @dev_priv: i915 private structure
1460  * @pipe: pipe to disable
1461  *
1462  * Disable @pipe, making sure that various hardware specific requirements
1463  * are met, if applicable, e.g. plane disabled, panel fitter off, etc.
1464  *
1465  * @pipe should be %PIPE_A or %PIPE_B.
1466  *
1467  * Will wait until the pipe has shut down before returning.
1468  */
1469 static void intel_disable_pipe(struct drm_i915_private *dev_priv,
1470                                enum pipe pipe)
1471 {
1472         int reg;
1473         u32 val;
1474
1475         /*
1476          * Make sure planes won't keep trying to pump pixels to us,
1477          * or we might hang the display.
1478          */
1479         assert_planes_disabled(dev_priv, pipe);
1480
1481         /* Don't disable pipe A or pipe A PLLs if needed */
1482         if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
1483                 return;
1484
1485         reg = PIPECONF(pipe);
1486         val = I915_READ(reg);
1487         if ((val & PIPECONF_ENABLE) == 0)
1488                 return;
1489
1490         I915_WRITE(reg, val & ~PIPECONF_ENABLE);
1491         intel_wait_for_pipe_off(dev_priv->dev, pipe);
1492 }
1493
1494 /*
1495  * Plane regs are double buffered, going from enabled->disabled needs a
1496  * trigger in order to latch.  The display address reg provides this.
1497  */
1498 static void intel_flush_display_plane(struct drm_i915_private *dev_priv,
1499                                       enum plane plane)
1500 {
1501         I915_WRITE(DSPADDR(plane), I915_READ(DSPADDR(plane)));
1502         I915_WRITE(DSPSURF(plane), I915_READ(DSPSURF(plane)));
1503 }
1504
1505 /**
1506  * intel_enable_plane - enable a display plane on a given pipe
1507  * @dev_priv: i915 private structure
1508  * @plane: plane to enable
1509  * @pipe: pipe being fed
1510  *
1511  * Enable @plane on @pipe, making sure that @pipe is running first.
1512  */
1513 static void intel_enable_plane(struct drm_i915_private *dev_priv,
1514                                enum plane plane, enum pipe pipe)
1515 {
1516         int reg;
1517         u32 val;
1518
1519         /* If the pipe isn't enabled, we can't pump pixels and may hang */
1520         assert_pipe_enabled(dev_priv, pipe);
1521
1522         reg = DSPCNTR(plane);
1523         val = I915_READ(reg);
1524         if (val & DISPLAY_PLANE_ENABLE)
1525                 return;
1526
1527         I915_WRITE(reg, val | DISPLAY_PLANE_ENABLE);
1528         intel_flush_display_plane(dev_priv, plane);
1529         intel_wait_for_vblank(dev_priv->dev, pipe);
1530 }
1531
1532 /**
1533  * intel_disable_plane - disable a display plane
1534  * @dev_priv: i915 private structure
1535  * @plane: plane to disable
1536  * @pipe: pipe consuming the data
1537  *
1538  * Disable @plane; should be an independent operation.
1539  */
1540 static void intel_disable_plane(struct drm_i915_private *dev_priv,
1541                                 enum plane plane, enum pipe pipe)
1542 {
1543         int reg;
1544         u32 val;
1545
1546         reg = DSPCNTR(plane);
1547         val = I915_READ(reg);
1548         if ((val & DISPLAY_PLANE_ENABLE) == 0)
1549                 return;
1550
1551         I915_WRITE(reg, val & ~DISPLAY_PLANE_ENABLE);
1552         intel_flush_display_plane(dev_priv, plane);
1553         intel_wait_for_vblank(dev_priv->dev, pipe);
1554 }
1555
1556 static void disable_pch_dp(struct drm_i915_private *dev_priv,
1557                            enum pipe pipe, int reg, u32 port_sel)
1558 {
1559         u32 val = I915_READ(reg);
1560         if (dp_pipe_enabled(dev_priv, pipe, port_sel, val)) {
1561                 DRM_DEBUG_KMS("Disabling pch dp %x on pipe %d\n", reg, pipe);
1562                 I915_WRITE(reg, val & ~DP_PORT_EN);
1563         }
1564 }
1565
1566 static void disable_pch_hdmi(struct drm_i915_private *dev_priv,
1567                              enum pipe pipe, int reg)
1568 {
1569         u32 val = I915_READ(reg);
1570         if (hdmi_pipe_enabled(dev_priv, val, pipe)) {
1571                 DRM_DEBUG_KMS("Disabling pch HDMI %x on pipe %d\n",
1572                               reg, pipe);
1573                 I915_WRITE(reg, val & ~PORT_ENABLE);
1574         }
1575 }
1576
1577 /* Disable any ports connected to this transcoder */
1578 static void intel_disable_pch_ports(struct drm_i915_private *dev_priv,
1579                                     enum pipe pipe)
1580 {
1581         u32 reg, val;
1582
1583         val = I915_READ(PCH_PP_CONTROL);
1584         I915_WRITE(PCH_PP_CONTROL, val | PANEL_UNLOCK_REGS);
1585
1586         disable_pch_dp(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
1587         disable_pch_dp(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
1588         disable_pch_dp(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
1589
1590         reg = PCH_ADPA;
1591         val = I915_READ(reg);
1592         if (adpa_pipe_enabled(dev_priv, val, pipe))
1593                 I915_WRITE(reg, val & ~ADPA_DAC_ENABLE);
1594
1595         reg = PCH_LVDS;
1596         val = I915_READ(reg);
1597         if (lvds_pipe_enabled(dev_priv, val, pipe)) {
1598                 DRM_DEBUG_KMS("disable lvds on pipe %d val 0x%08x\n", pipe, val);
1599                 I915_WRITE(reg, val & ~LVDS_PORT_EN);
1600                 POSTING_READ(reg);
1601                 udelay(100);
1602         }
1603
1604         disable_pch_hdmi(dev_priv, pipe, HDMIB);
1605         disable_pch_hdmi(dev_priv, pipe, HDMIC);
1606         disable_pch_hdmi(dev_priv, pipe, HDMID);
1607 }
1608
1609 static void i8xx_disable_fbc(struct drm_device *dev)
1610 {
1611         struct drm_i915_private *dev_priv = dev->dev_private;
1612         u32 fbc_ctl;
1613
1614         /* Disable compression */
1615         fbc_ctl = I915_READ(FBC_CONTROL);
1616         if ((fbc_ctl & FBC_CTL_EN) == 0)
1617                 return;
1618
1619         fbc_ctl &= ~FBC_CTL_EN;
1620         I915_WRITE(FBC_CONTROL, fbc_ctl);
1621
1622         /* Wait for compressing bit to clear */
1623         if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
1624                 DRM_DEBUG_KMS("FBC idle timed out\n");
1625                 return;
1626         }
1627
1628         DRM_DEBUG_KMS("disabled FBC\n");
1629 }
1630
1631 static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1632 {
1633         struct drm_device *dev = crtc->dev;
1634         struct drm_i915_private *dev_priv = dev->dev_private;
1635         struct drm_framebuffer *fb = crtc->fb;
1636         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1637         struct drm_i915_gem_object *obj = intel_fb->obj;
1638         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1639         int cfb_pitch;
1640         int plane, i;
1641         u32 fbc_ctl, fbc_ctl2;
1642
1643         cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
1644         if (fb->pitches[0] < cfb_pitch)
1645                 cfb_pitch = fb->pitches[0];
1646
1647         /* FBC_CTL wants 64B units */
1648         cfb_pitch = (cfb_pitch / 64) - 1;
1649         plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
1650
1651         /* Clear old tags */
1652         for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
1653                 I915_WRITE(FBC_TAG + (i * 4), 0);
1654
1655         /* Set it up... */
1656         fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
1657         fbc_ctl2 |= plane;
1658         I915_WRITE(FBC_CONTROL2, fbc_ctl2);
1659         I915_WRITE(FBC_FENCE_OFF, crtc->y);
1660
1661         /* enable it... */
1662         fbc_ctl = FBC_CTL_EN | FBC_CTL_PERIODIC;
1663         if (IS_I945GM(dev))
1664                 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
1665         fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
1666         fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
1667         fbc_ctl |= obj->fence_reg;
1668         I915_WRITE(FBC_CONTROL, fbc_ctl);
1669
1670         DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
1671                       cfb_pitch, crtc->y, intel_crtc->plane);
1672 }
1673
1674 static bool i8xx_fbc_enabled(struct drm_device *dev)
1675 {
1676         struct drm_i915_private *dev_priv = dev->dev_private;
1677
1678         return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
1679 }
1680
1681 static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1682 {
1683         struct drm_device *dev = crtc->dev;
1684         struct drm_i915_private *dev_priv = dev->dev_private;
1685         struct drm_framebuffer *fb = crtc->fb;
1686         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1687         struct drm_i915_gem_object *obj = intel_fb->obj;
1688         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1689         int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
1690         unsigned long stall_watermark = 200;
1691         u32 dpfc_ctl;
1692
1693         dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
1694         dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
1695         I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
1696
1697         I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
1698                    (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
1699                    (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
1700         I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
1701
1702         /* enable it... */
1703         I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN);
1704
1705         DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
1706 }
1707
1708 static void g4x_disable_fbc(struct drm_device *dev)
1709 {
1710         struct drm_i915_private *dev_priv = dev->dev_private;
1711         u32 dpfc_ctl;
1712
1713         /* Disable compression */
1714         dpfc_ctl = I915_READ(DPFC_CONTROL);
1715         if (dpfc_ctl & DPFC_CTL_EN) {
1716                 dpfc_ctl &= ~DPFC_CTL_EN;
1717                 I915_WRITE(DPFC_CONTROL, dpfc_ctl);
1718
1719                 DRM_DEBUG_KMS("disabled FBC\n");
1720         }
1721 }
1722
1723 static bool g4x_fbc_enabled(struct drm_device *dev)
1724 {
1725         struct drm_i915_private *dev_priv = dev->dev_private;
1726
1727         return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
1728 }
1729
1730 static void sandybridge_blit_fbc_update(struct drm_device *dev)
1731 {
1732         struct drm_i915_private *dev_priv = dev->dev_private;
1733         u32 blt_ecoskpd;
1734
1735         /* Make sure blitter notifies FBC of writes */
1736         gen6_gt_force_wake_get(dev_priv);
1737         blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
1738         blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY <<
1739                 GEN6_BLITTER_LOCK_SHIFT;
1740         I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
1741         blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY;
1742         I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
1743         blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
1744                          GEN6_BLITTER_LOCK_SHIFT);
1745         I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
1746         POSTING_READ(GEN6_BLITTER_ECOSKPD);
1747         gen6_gt_force_wake_put(dev_priv);
1748 }
1749
1750 static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1751 {
1752         struct drm_device *dev = crtc->dev;
1753         struct drm_i915_private *dev_priv = dev->dev_private;
1754         struct drm_framebuffer *fb = crtc->fb;
1755         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1756         struct drm_i915_gem_object *obj = intel_fb->obj;
1757         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1758         int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
1759         unsigned long stall_watermark = 200;
1760         u32 dpfc_ctl;
1761
1762         dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
1763         dpfc_ctl &= DPFC_RESERVED;
1764         dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
1765         /* Set persistent mode for front-buffer rendering, ala X. */
1766         dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE;
1767         dpfc_ctl |= (DPFC_CTL_FENCE_EN | obj->fence_reg);
1768         I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
1769
1770         I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
1771                    (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
1772                    (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
1773         I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
1774         I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset | ILK_FBC_RT_VALID);
1775         /* enable it... */
1776         I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
1777
1778         if (IS_GEN6(dev)) {
1779                 I915_WRITE(SNB_DPFC_CTL_SA,
1780                            SNB_CPU_FENCE_ENABLE | obj->fence_reg);
1781                 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
1782                 sandybridge_blit_fbc_update(dev);
1783         }
1784
1785         DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
1786 }
1787
1788 static void ironlake_disable_fbc(struct drm_device *dev)
1789 {
1790         struct drm_i915_private *dev_priv = dev->dev_private;
1791         u32 dpfc_ctl;
1792
1793         /* Disable compression */
1794         dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
1795         if (dpfc_ctl & DPFC_CTL_EN) {
1796                 dpfc_ctl &= ~DPFC_CTL_EN;
1797                 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
1798
1799                 DRM_DEBUG_KMS("disabled FBC\n");
1800         }
1801 }
1802
1803 static bool ironlake_fbc_enabled(struct drm_device *dev)
1804 {
1805         struct drm_i915_private *dev_priv = dev->dev_private;
1806
1807         return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
1808 }
1809
1810 bool intel_fbc_enabled(struct drm_device *dev)
1811 {
1812         struct drm_i915_private *dev_priv = dev->dev_private;
1813
1814         if (!dev_priv->display.fbc_enabled)
1815                 return false;
1816
1817         return dev_priv->display.fbc_enabled(dev);
1818 }
1819
1820 static void intel_fbc_work_fn(struct work_struct *__work)
1821 {
1822         struct intel_fbc_work *work =
1823                 container_of(to_delayed_work(__work),
1824                              struct intel_fbc_work, work);
1825         struct drm_device *dev = work->crtc->dev;
1826         struct drm_i915_private *dev_priv = dev->dev_private;
1827
1828         mutex_lock(&dev->struct_mutex);
1829         if (work == dev_priv->fbc_work) {
1830                 /* Double check that we haven't switched fb without cancelling
1831                  * the prior work.
1832                  */
1833                 if (work->crtc->fb == work->fb) {
1834                         dev_priv->display.enable_fbc(work->crtc,
1835                                                      work->interval);
1836
1837                         dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
1838                         dev_priv->cfb_fb = work->crtc->fb->base.id;
1839                         dev_priv->cfb_y = work->crtc->y;
1840                 }
1841
1842                 dev_priv->fbc_work = NULL;
1843         }
1844         mutex_unlock(&dev->struct_mutex);
1845
1846         kfree(work);
1847 }
1848
1849 static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
1850 {
1851         if (dev_priv->fbc_work == NULL)
1852                 return;
1853
1854         DRM_DEBUG_KMS("cancelling pending FBC enable\n");
1855
1856         /* Synchronisation is provided by struct_mutex and checking of
1857          * dev_priv->fbc_work, so we can perform the cancellation
1858          * entirely asynchronously.
1859          */
1860         if (cancel_delayed_work(&dev_priv->fbc_work->work))
1861                 /* tasklet was killed before being run, clean up */
1862                 kfree(dev_priv->fbc_work);
1863
1864         /* Mark the work as no longer wanted so that if it does
1865          * wake-up (because the work was already running and waiting
1866          * for our mutex), it will discover that is no longer
1867          * necessary to run.
1868          */
1869         dev_priv->fbc_work = NULL;
1870 }
1871
1872 static void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1873 {
1874         struct intel_fbc_work *work;
1875         struct drm_device *dev = crtc->dev;
1876         struct drm_i915_private *dev_priv = dev->dev_private;
1877
1878         if (!dev_priv->display.enable_fbc)
1879                 return;
1880
1881         intel_cancel_fbc_work(dev_priv);
1882
1883         work = kzalloc(sizeof *work, GFP_KERNEL);
1884         if (work == NULL) {
1885                 dev_priv->display.enable_fbc(crtc, interval);
1886                 return;
1887         }
1888
1889         work->crtc = crtc;
1890         work->fb = crtc->fb;
1891         work->interval = interval;
1892         INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
1893
1894         dev_priv->fbc_work = work;
1895
1896         DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
1897
1898         /* Delay the actual enabling to let pageflipping cease and the
1899          * display to settle before starting the compression. Note that
1900          * this delay also serves a second purpose: it allows for a
1901          * vblank to pass after disabling the FBC before we attempt
1902          * to modify the control registers.
1903          *
1904          * A more complicated solution would involve tracking vblanks
1905          * following the termination of the page-flipping sequence
1906          * and indeed performing the enable as a co-routine and not
1907          * waiting synchronously upon the vblank.
1908          */
1909         schedule_delayed_work(&work->work, msecs_to_jiffies(50));
1910 }
1911
1912 void intel_disable_fbc(struct drm_device *dev)
1913 {
1914         struct drm_i915_private *dev_priv = dev->dev_private;
1915
1916         intel_cancel_fbc_work(dev_priv);
1917
1918         if (!dev_priv->display.disable_fbc)
1919                 return;
1920
1921         dev_priv->display.disable_fbc(dev);
1922         dev_priv->cfb_plane = -1;
1923 }
1924
1925 /**
1926  * intel_update_fbc - enable/disable FBC as needed
1927  * @dev: the drm_device
1928  *
1929  * Set up the framebuffer compression hardware at mode set time.  We
1930  * enable it if possible:
1931  *   - plane A only (on pre-965)
1932  *   - no pixel mulitply/line duplication
1933  *   - no alpha buffer discard
1934  *   - no dual wide
1935  *   - framebuffer <= 2048 in width, 1536 in height
1936  *
1937  * We can't assume that any compression will take place (worst case),
1938  * so the compressed buffer has to be the same size as the uncompressed
1939  * one.  It also must reside (along with the line length buffer) in
1940  * stolen memory.
1941  *
1942  * We need to enable/disable FBC on a global basis.
1943  */
1944 static void intel_update_fbc(struct drm_device *dev)
1945 {
1946         struct drm_i915_private *dev_priv = dev->dev_private;
1947         struct drm_crtc *crtc = NULL, *tmp_crtc;
1948         struct intel_crtc *intel_crtc;
1949         struct drm_framebuffer *fb;
1950         struct intel_framebuffer *intel_fb;
1951         struct drm_i915_gem_object *obj;
1952         int enable_fbc;
1953
1954         DRM_DEBUG_KMS("\n");
1955
1956         if (!i915_powersave)
1957                 return;
1958
1959         if (!I915_HAS_FBC(dev))
1960                 return;
1961
1962         /*
1963          * If FBC is already on, we just have to verify that we can
1964          * keep it that way...
1965          * Need to disable if:
1966          *   - more than one pipe is active
1967          *   - changing FBC params (stride, fence, mode)
1968          *   - new fb is too large to fit in compressed buffer
1969          *   - going to an unsupported config (interlace, pixel multiply, etc.)
1970          */
1971         list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
1972                 if (tmp_crtc->enabled && tmp_crtc->fb) {
1973                         if (crtc) {
1974                                 DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
1975                                 dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
1976                                 goto out_disable;
1977                         }
1978                         crtc = tmp_crtc;
1979                 }
1980         }
1981
1982         if (!crtc || crtc->fb == NULL) {
1983                 DRM_DEBUG_KMS("no output, disabling\n");
1984                 dev_priv->no_fbc_reason = FBC_NO_OUTPUT;
1985                 goto out_disable;
1986         }
1987
1988         intel_crtc = to_intel_crtc(crtc);
1989         fb = crtc->fb;
1990         intel_fb = to_intel_framebuffer(fb);
1991         obj = intel_fb->obj;
1992
1993         enable_fbc = i915_enable_fbc;
1994         if (enable_fbc < 0) {
1995                 DRM_DEBUG_KMS("fbc set to per-chip default\n");
1996                 enable_fbc = 1;
1997                 if (INTEL_INFO(dev)->gen <= 6)
1998                         enable_fbc = 0;
1999         }
2000         if (!enable_fbc) {
2001                 DRM_DEBUG_KMS("fbc disabled per module param\n");
2002                 dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
2003                 goto out_disable;
2004         }
2005         if (intel_fb->obj->base.size > dev_priv->cfb_size) {
2006                 DRM_DEBUG_KMS("framebuffer too large, disabling "
2007                               "compression\n");
2008                 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
2009                 goto out_disable;
2010         }
2011         if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) ||
2012             (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
2013                 DRM_DEBUG_KMS("mode incompatible with compression, "
2014                               "disabling\n");
2015                 dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
2016                 goto out_disable;
2017         }
2018         if ((crtc->mode.hdisplay > 2048) ||
2019             (crtc->mode.vdisplay > 1536)) {
2020                 DRM_DEBUG_KMS("mode too large for compression, disabling\n");
2021                 dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
2022                 goto out_disable;
2023         }
2024         if ((IS_I915GM(dev) || IS_I945GM(dev)) && intel_crtc->plane != 0) {
2025                 DRM_DEBUG_KMS("plane not 0, disabling compression\n");
2026                 dev_priv->no_fbc_reason = FBC_BAD_PLANE;
2027                 goto out_disable;
2028         }
2029
2030         /* The use of a CPU fence is mandatory in order to detect writes
2031          * by the CPU to the scanout and trigger updates to the FBC.
2032          */
2033         if (obj->tiling_mode != I915_TILING_X ||
2034             obj->fence_reg == I915_FENCE_REG_NONE) {
2035                 DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
2036                 dev_priv->no_fbc_reason = FBC_NOT_TILED;
2037                 goto out_disable;
2038         }
2039
2040         /* If the kernel debugger is active, always disable compression */
2041         if (in_dbg_master())
2042                 goto out_disable;
2043
2044         /* If the scanout has not changed, don't modify the FBC settings.
2045          * Note that we make the fundamental assumption that the fb->obj
2046          * cannot be unpinned (and have its GTT offset and fence revoked)
2047          * without first being decoupled from the scanout and FBC disabled.
2048          */
2049         if (dev_priv->cfb_plane == intel_crtc->plane &&
2050             dev_priv->cfb_fb == fb->base.id &&
2051             dev_priv->cfb_y == crtc->y)
2052                 return;
2053
2054         if (intel_fbc_enabled(dev)) {
2055                 /* We update FBC along two paths, after changing fb/crtc
2056                  * configuration (modeswitching) and after page-flipping
2057                  * finishes. For the latter, we know that not only did
2058                  * we disable the FBC at the start of the page-flip
2059                  * sequence, but also more than one vblank has passed.
2060                  *
2061                  * For the former case of modeswitching, it is possible
2062                  * to switch between two FBC valid configurations
2063                  * instantaneously so we do need to disable the FBC
2064                  * before we can modify its control registers. We also
2065                  * have to wait for the next vblank for that to take
2066                  * effect. However, since we delay enabling FBC we can
2067                  * assume that a vblank has passed since disabling and
2068                  * that we can safely alter the registers in the deferred
2069                  * callback.
2070                  *
2071                  * In the scenario that we go from a valid to invalid
2072                  * and then back to valid FBC configuration we have
2073                  * no strict enforcement that a vblank occurred since
2074                  * disabling the FBC. However, along all current pipe
2075                  * disabling paths we do need to wait for a vblank at
2076                  * some point. And we wait before enabling FBC anyway.
2077                  */
2078                 DRM_DEBUG_KMS("disabling active FBC for update\n");
2079                 intel_disable_fbc(dev);
2080         }
2081
2082         intel_enable_fbc(crtc, 500);
2083         return;
2084
2085 out_disable:
2086         /* Multiple disables should be harmless */
2087         if (intel_fbc_enabled(dev)) {
2088                 DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
2089                 intel_disable_fbc(dev);
2090         }
2091 }
2092
2093 int
2094 intel_pin_and_fence_fb_obj(struct drm_device *dev,
2095                            struct drm_i915_gem_object *obj,
2096                            struct intel_ring_buffer *pipelined)
2097 {
2098         struct drm_i915_private *dev_priv = dev->dev_private;
2099         u32 alignment;
2100         int ret;
2101
2102         switch (obj->tiling_mode) {
2103         case I915_TILING_NONE:
2104                 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
2105                         alignment = 128 * 1024;
2106                 else if (INTEL_INFO(dev)->gen >= 4)
2107                         alignment = 4 * 1024;
2108                 else
2109                         alignment = 64 * 1024;
2110                 break;
2111         case I915_TILING_X:
2112                 /* pin() will align the object as required by fence */
2113                 alignment = 0;
2114                 break;
2115         case I915_TILING_Y:
2116                 /* FIXME: Is this true? */
2117                 DRM_ERROR("Y tiled not allowed for scan out buffers\n");
2118                 return -EINVAL;
2119         default:
2120                 BUG();
2121         }
2122
2123         dev_priv->mm.interruptible = false;
2124         ret = i915_gem_object_pin_to_display_plane(obj, alignment, pipelined);
2125         if (ret)
2126                 goto err_interruptible;
2127
2128         /* Install a fence for tiled scan-out. Pre-i965 always needs a
2129          * fence, whereas 965+ only requires a fence if using
2130          * framebuffer compression.  For simplicity, we always install
2131          * a fence as the cost is not that onerous.
2132          */
2133         if (obj->tiling_mode != I915_TILING_NONE) {
2134                 ret = i915_gem_object_get_fence(obj, pipelined);
2135                 if (ret)
2136                         goto err_unpin;
2137
2138                 i915_gem_object_pin_fence(obj);
2139         }
2140
2141         dev_priv->mm.interruptible = true;
2142         return 0;
2143
2144 err_unpin:
2145         i915_gem_object_unpin(obj);
2146 err_interruptible:
2147         dev_priv->mm.interruptible = true;
2148         return ret;
2149 }
2150
2151 void intel_unpin_fb_obj(struct drm_i915_gem_object *obj)
2152 {
2153         i915_gem_object_unpin_fence(obj);
2154         i915_gem_object_unpin(obj);
2155 }
2156
2157 static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb,
2158                              int x, int y)
2159 {
2160         struct drm_device *dev = crtc->dev;
2161         struct drm_i915_private *dev_priv = dev->dev_private;
2162         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2163         struct intel_framebuffer *intel_fb;
2164         struct drm_i915_gem_object *obj;
2165         int plane = intel_crtc->plane;
2166         unsigned long Start, Offset;
2167         u32 dspcntr;
2168         u32 reg;
2169
2170         switch (plane) {
2171         case 0:
2172         case 1:
2173                 break;
2174         default:
2175                 DRM_ERROR("Can't update plane %d in SAREA\n", plane);
2176                 return -EINVAL;
2177         }
2178
2179         intel_fb = to_intel_framebuffer(fb);
2180         obj = intel_fb->obj;
2181
2182         reg = DSPCNTR(plane);
2183         dspcntr = I915_READ(reg);
2184         /* Mask out pixel format bits in case we change it */
2185         dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
2186         switch (fb->bits_per_pixel) {
2187         case 8:
2188                 dspcntr |= DISPPLANE_8BPP;
2189                 break;
2190         case 16:
2191                 if (fb->depth == 15)
2192                         dspcntr |= DISPPLANE_15_16BPP;
2193                 else
2194                         dspcntr |= DISPPLANE_16BPP;
2195                 break;
2196         case 24:
2197         case 32:
2198                 dspcntr |= DISPPLANE_32BPP_NO_ALPHA;
2199                 break;
2200         default:
2201                 DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel);
2202                 return -EINVAL;
2203         }
2204         if (INTEL_INFO(dev)->gen >= 4) {
2205                 if (obj->tiling_mode != I915_TILING_NONE)
2206                         dspcntr |= DISPPLANE_TILED;
2207                 else
2208                         dspcntr &= ~DISPPLANE_TILED;
2209         }
2210
2211         I915_WRITE(reg, dspcntr);
2212
2213         Start = obj->gtt_offset;
2214         Offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8);
2215
2216         DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
2217                       Start, Offset, x, y, fb->pitches[0]);
2218         I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
2219         if (INTEL_INFO(dev)->gen >= 4) {
2220                 I915_WRITE(DSPSURF(plane), Start);
2221                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2222                 I915_WRITE(DSPADDR(plane), Offset);
2223         } else
2224                 I915_WRITE(DSPADDR(plane), Start + Offset);
2225         POSTING_READ(reg);
2226
2227         return 0;
2228 }
2229
2230 static int ironlake_update_plane(struct drm_crtc *crtc,
2231                                  struct drm_framebuffer *fb, int x, int y)
2232 {
2233         struct drm_device *dev = crtc->dev;
2234         struct drm_i915_private *dev_priv = dev->dev_private;
2235         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2236         struct intel_framebuffer *intel_fb;
2237         struct drm_i915_gem_object *obj;
2238         int plane = intel_crtc->plane;
2239         unsigned long Start, Offset;
2240         u32 dspcntr;
2241         u32 reg;
2242
2243         switch (plane) {
2244         case 0:
2245         case 1:
2246         case 2:
2247                 break;
2248         default:
2249                 DRM_ERROR("Can't update plane %d in SAREA\n", plane);
2250                 return -EINVAL;
2251         }
2252
2253         intel_fb = to_intel_framebuffer(fb);
2254         obj = intel_fb->obj;
2255
2256         reg = DSPCNTR(plane);
2257         dspcntr = I915_READ(reg);
2258         /* Mask out pixel format bits in case we change it */
2259         dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
2260         switch (fb->bits_per_pixel) {
2261         case 8:
2262                 dspcntr |= DISPPLANE_8BPP;
2263                 break;
2264         case 16:
2265                 if (fb->depth != 16)
2266                         return -EINVAL;
2267
2268                 dspcntr |= DISPPLANE_16BPP;
2269                 break;
2270         case 24:
2271         case 32:
2272                 if (fb->depth == 24)
2273                         dspcntr |= DISPPLANE_32BPP_NO_ALPHA;
2274                 else if (fb->depth == 30)
2275                         dspcntr |= DISPPLANE_32BPP_30BIT_NO_ALPHA;
2276                 else
2277                         return -EINVAL;
2278                 break;
2279         default:
2280                 DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel);
2281                 return -EINVAL;
2282         }
2283
2284         if (obj->tiling_mode != I915_TILING_NONE)
2285                 dspcntr |= DISPPLANE_TILED;
2286         else
2287                 dspcntr &= ~DISPPLANE_TILED;
2288
2289         /* must disable */
2290         dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
2291
2292         I915_WRITE(reg, dspcntr);
2293
2294         Start = obj->gtt_offset;
2295         Offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8);
2296
2297         DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
2298                       Start, Offset, x, y, fb->pitches[0]);
2299         I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
2300         I915_WRITE(DSPSURF(plane), Start);
2301         I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2302         I915_WRITE(DSPADDR(plane), Offset);
2303         POSTING_READ(reg);
2304
2305         return 0;
2306 }
2307
2308 /* Assume fb object is pinned & idle & fenced and just update base pointers */
2309 static int
2310 intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
2311                            int x, int y, enum mode_set_atomic state)
2312 {
2313         struct drm_device *dev = crtc->dev;
2314         struct drm_i915_private *dev_priv = dev->dev_private;
2315         int ret;
2316
2317         ret = dev_priv->display.update_plane(crtc, fb, x, y);
2318         if (ret)
2319                 return ret;
2320
2321         intel_update_fbc(dev);
2322         intel_increase_pllclock(crtc);
2323
2324         return 0;
2325 }
2326
2327 static int
2328 intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
2329                     struct drm_framebuffer *old_fb)
2330 {
2331         struct drm_device *dev = crtc->dev;
2332         struct drm_i915_master_private *master_priv;
2333         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2334         int ret;
2335
2336         /* no fb bound */
2337         if (!crtc->fb) {
2338                 DRM_ERROR("No FB bound\n");
2339                 return 0;
2340         }
2341
2342         switch (intel_crtc->plane) {
2343         case 0:
2344         case 1:
2345                 break;
2346         case 2:
2347                 if (IS_IVYBRIDGE(dev))
2348                         break;
2349                 /* fall through otherwise */
2350         default:
2351                 DRM_ERROR("no plane for crtc\n");
2352                 return -EINVAL;
2353         }
2354
2355         mutex_lock(&dev->struct_mutex);
2356         ret = intel_pin_and_fence_fb_obj(dev,
2357                                          to_intel_framebuffer(crtc->fb)->obj,
2358                                          NULL);
2359         if (ret != 0) {
2360                 mutex_unlock(&dev->struct_mutex);
2361                 DRM_ERROR("pin & fence failed\n");
2362                 return ret;
2363         }
2364
2365         if (old_fb) {
2366                 struct drm_i915_private *dev_priv = dev->dev_private;
2367                 struct drm_i915_gem_object *obj = to_intel_framebuffer(old_fb)->obj;
2368
2369                 wait_event(dev_priv->pending_flip_queue,
2370                            atomic_read(&dev_priv->mm.wedged) ||
2371                            atomic_read(&obj->pending_flip) == 0);
2372
2373                 /* Big Hammer, we also need to ensure that any pending
2374                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
2375                  * current scanout is retired before unpinning the old
2376                  * framebuffer.
2377                  *
2378                  * This should only fail upon a hung GPU, in which case we
2379                  * can safely continue.
2380                  */
2381                 ret = i915_gem_object_finish_gpu(obj);
2382                 (void) ret;
2383         }
2384
2385         ret = intel_pipe_set_base_atomic(crtc, crtc->fb, x, y,
2386                                          LEAVE_ATOMIC_MODE_SET);
2387         if (ret) {
2388                 intel_unpin_fb_obj(to_intel_framebuffer(crtc->fb)->obj);
2389                 mutex_unlock(&dev->struct_mutex);
2390                 DRM_ERROR("failed to update base address\n");
2391                 return ret;
2392         }
2393
2394         if (old_fb) {
2395                 intel_wait_for_vblank(dev, intel_crtc->pipe);
2396                 intel_unpin_fb_obj(to_intel_framebuffer(old_fb)->obj);
2397         }
2398
2399         mutex_unlock(&dev->struct_mutex);
2400
2401         if (!dev->primary->master)
2402                 return 0;
2403
2404         master_priv = dev->primary->master->driver_priv;
2405         if (!master_priv->sarea_priv)
2406                 return 0;
2407
2408         if (intel_crtc->pipe) {
2409                 master_priv->sarea_priv->pipeB_x = x;
2410                 master_priv->sarea_priv->pipeB_y = y;
2411         } else {
2412                 master_priv->sarea_priv->pipeA_x = x;
2413                 master_priv->sarea_priv->pipeA_y = y;
2414         }
2415
2416         return 0;
2417 }
2418
2419 static void ironlake_set_pll_edp(struct drm_crtc *crtc, int clock)
2420 {
2421         struct drm_device *dev = crtc->dev;
2422         struct drm_i915_private *dev_priv = dev->dev_private;
2423         u32 dpa_ctl;
2424
2425         DRM_DEBUG_KMS("eDP PLL enable for clock %d\n", clock);
2426         dpa_ctl = I915_READ(DP_A);
2427         dpa_ctl &= ~DP_PLL_FREQ_MASK;
2428
2429         if (clock < 200000) {
2430                 u32 temp;
2431                 dpa_ctl |= DP_PLL_FREQ_160MHZ;
2432                 /* workaround for 160Mhz:
2433                    1) program 0x4600c bits 15:0 = 0x8124
2434                    2) program 0x46010 bit 0 = 1
2435                    3) program 0x46034 bit 24 = 1
2436                    4) program 0x64000 bit 14 = 1
2437                    */
2438                 temp = I915_READ(0x4600c);
2439                 temp &= 0xffff0000;
2440                 I915_WRITE(0x4600c, temp | 0x8124);
2441
2442                 temp = I915_READ(0x46010);
2443                 I915_WRITE(0x46010, temp | 1);
2444
2445                 temp = I915_READ(0x46034);
2446                 I915_WRITE(0x46034, temp | (1 << 24));
2447         } else {
2448                 dpa_ctl |= DP_PLL_FREQ_270MHZ;
2449         }
2450         I915_WRITE(DP_A, dpa_ctl);
2451
2452         POSTING_READ(DP_A);
2453         udelay(500);
2454 }
2455
2456 static void intel_fdi_normal_train(struct drm_crtc *crtc)
2457 {
2458         struct drm_device *dev = crtc->dev;
2459         struct drm_i915_private *dev_priv = dev->dev_private;
2460         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2461         int pipe = intel_crtc->pipe;
2462         u32 reg, temp;
2463
2464         /* enable normal train */
2465         reg = FDI_TX_CTL(pipe);
2466         temp = I915_READ(reg);
2467         if (IS_IVYBRIDGE(dev)) {
2468                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
2469                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
2470         } else {
2471                 temp &= ~FDI_LINK_TRAIN_NONE;
2472                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
2473         }
2474         I915_WRITE(reg, temp);
2475
2476         reg = FDI_RX_CTL(pipe);
2477         temp = I915_READ(reg);
2478         if (HAS_PCH_CPT(dev)) {
2479                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2480                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
2481         } else {
2482                 temp &= ~FDI_LINK_TRAIN_NONE;
2483                 temp |= FDI_LINK_TRAIN_NONE;
2484         }
2485         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
2486
2487         /* wait one idle pattern time */
2488         POSTING_READ(reg);
2489         udelay(1000);
2490
2491         /* IVB wants error correction enabled */
2492         if (IS_IVYBRIDGE(dev))
2493                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
2494                            FDI_FE_ERRC_ENABLE);
2495 }
2496
2497 static void cpt_phase_pointer_enable(struct drm_device *dev, int pipe)
2498 {
2499         struct drm_i915_private *dev_priv = dev->dev_private;
2500         u32 flags = I915_READ(SOUTH_CHICKEN1);
2501
2502         flags |= FDI_PHASE_SYNC_OVR(pipe);
2503         I915_WRITE(SOUTH_CHICKEN1, flags); /* once to unlock... */
2504         flags |= FDI_PHASE_SYNC_EN(pipe);
2505         I915_WRITE(SOUTH_CHICKEN1, flags); /* then again to enable */
2506         POSTING_READ(SOUTH_CHICKEN1);
2507 }
2508
2509 /* The FDI link training functions for ILK/Ibexpeak. */
2510 static void ironlake_fdi_link_train(struct drm_crtc *crtc)
2511 {
2512         struct drm_device *dev = crtc->dev;
2513         struct drm_i915_private *dev_priv = dev->dev_private;
2514         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2515         int pipe = intel_crtc->pipe;
2516         int plane = intel_crtc->plane;
2517         u32 reg, temp, tries;
2518
2519         /* FDI needs bits from pipe & plane first */
2520         assert_pipe_enabled(dev_priv, pipe);
2521         assert_plane_enabled(dev_priv, plane);
2522
2523         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
2524            for train result */
2525         reg = FDI_RX_IMR(pipe);
2526         temp = I915_READ(reg);
2527         temp &= ~FDI_RX_SYMBOL_LOCK;
2528         temp &= ~FDI_RX_BIT_LOCK;
2529         I915_WRITE(reg, temp);
2530         I915_READ(reg);
2531         udelay(150);
2532
2533         /* enable CPU FDI TX and PCH FDI RX */
2534         reg = FDI_TX_CTL(pipe);
2535         temp = I915_READ(reg);
2536         temp &= ~(7 << 19);
2537         temp |= (intel_crtc->fdi_lanes - 1) << 19;
2538         temp &= ~FDI_LINK_TRAIN_NONE;
2539         temp |= FDI_LINK_TRAIN_PATTERN_1;
2540         I915_WRITE(reg, temp | FDI_TX_ENABLE);
2541
2542         reg = FDI_RX_CTL(pipe);
2543         temp = I915_READ(reg);
2544         temp &= ~FDI_LINK_TRAIN_NONE;
2545         temp |= FDI_LINK_TRAIN_PATTERN_1;
2546         I915_WRITE(reg, temp | FDI_RX_ENABLE);
2547
2548         POSTING_READ(reg);
2549         udelay(150);
2550
2551         /* Ironlake workaround, enable clock pointer after FDI enable*/
2552         if (HAS_PCH_IBX(dev)) {
2553                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
2554                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
2555                            FDI_RX_PHASE_SYNC_POINTER_EN);
2556         }
2557
2558         reg = FDI_RX_IIR(pipe);
2559         for (tries = 0; tries < 5; tries++) {
2560                 temp = I915_READ(reg);
2561                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2562
2563                 if ((temp & FDI_RX_BIT_LOCK)) {
2564                         DRM_DEBUG_KMS("FDI train 1 done.\n");
2565                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
2566                         break;
2567                 }
2568         }
2569         if (tries == 5)
2570                 DRM_ERROR("FDI train 1 fail!\n");
2571
2572         /* Train 2 */
2573         reg = FDI_TX_CTL(pipe);
2574         temp = I915_READ(reg);
2575         temp &= ~FDI_LINK_TRAIN_NONE;
2576         temp |= FDI_LINK_TRAIN_PATTERN_2;
2577         I915_WRITE(reg, temp);
2578
2579         reg = FDI_RX_CTL(pipe);
2580         temp = I915_READ(reg);
2581         temp &= ~FDI_LINK_TRAIN_NONE;
2582         temp |= FDI_LINK_TRAIN_PATTERN_2;
2583         I915_WRITE(reg, temp);
2584
2585         POSTING_READ(reg);
2586         udelay(150);
2587
2588         reg = FDI_RX_IIR(pipe);
2589         for (tries = 0; tries < 5; tries++) {
2590                 temp = I915_READ(reg);
2591                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2592
2593                 if (temp & FDI_RX_SYMBOL_LOCK) {
2594                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
2595                         DRM_DEBUG_KMS("FDI train 2 done.\n");
2596                         break;
2597                 }
2598         }
2599         if (tries == 5)
2600                 DRM_ERROR("FDI train 2 fail!\n");
2601
2602         DRM_DEBUG_KMS("FDI train done\n");
2603
2604 }
2605
2606 static const int snb_b_fdi_train_param[] = {
2607         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
2608         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
2609         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
2610         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
2611 };
2612
2613 /* The FDI link training functions for SNB/Cougarpoint. */
2614 static void gen6_fdi_link_train(struct drm_crtc *crtc)
2615 {
2616         struct drm_device *dev = crtc->dev;
2617         struct drm_i915_private *dev_priv = dev->dev_private;
2618         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2619         int pipe = intel_crtc->pipe;
2620         u32 reg, temp, i, retry;
2621
2622         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
2623            for train result */
2624         reg = FDI_RX_IMR(pipe);
2625         temp = I915_READ(reg);
2626         temp &= ~FDI_RX_SYMBOL_LOCK;
2627         temp &= ~FDI_RX_BIT_LOCK;
2628         I915_WRITE(reg, temp);
2629
2630         POSTING_READ(reg);
2631         udelay(150);
2632
2633         /* enable CPU FDI TX and PCH FDI RX */
2634         reg = FDI_TX_CTL(pipe);
2635         temp = I915_READ(reg);
2636         temp &= ~(7 << 19);
2637         temp |= (intel_crtc->fdi_lanes - 1) << 19;
2638         temp &= ~FDI_LINK_TRAIN_NONE;
2639         temp |= FDI_LINK_TRAIN_PATTERN_1;
2640         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2641         /* SNB-B */
2642         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
2643         I915_WRITE(reg, temp | FDI_TX_ENABLE);
2644
2645         reg = FDI_RX_CTL(pipe);
2646         temp = I915_READ(reg);
2647         if (HAS_PCH_CPT(dev)) {
2648                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2649                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
2650         } else {
2651                 temp &= ~FDI_LINK_TRAIN_NONE;
2652                 temp |= FDI_LINK_TRAIN_PATTERN_1;
2653         }
2654         I915_WRITE(reg, temp | FDI_RX_ENABLE);
2655
2656         POSTING_READ(reg);
2657         udelay(150);
2658
2659         if (HAS_PCH_CPT(dev))
2660                 cpt_phase_pointer_enable(dev, pipe);
2661
2662         for (i = 0; i < 4; i++) {
2663                 reg = FDI_TX_CTL(pipe);
2664                 temp = I915_READ(reg);
2665                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2666                 temp |= snb_b_fdi_train_param[i];
2667                 I915_WRITE(reg, temp);
2668
2669                 POSTING_READ(reg);
2670                 udelay(500);
2671
2672                 for (retry = 0; retry < 5; retry++) {
2673                         reg = FDI_RX_IIR(pipe);
2674                         temp = I915_READ(reg);
2675                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2676                         if (temp & FDI_RX_BIT_LOCK) {
2677                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
2678                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
2679                                 break;
2680                         }
2681                         udelay(50);
2682                 }
2683                 if (retry < 5)
2684                         break;
2685         }
2686         if (i == 4)
2687                 DRM_ERROR("FDI train 1 fail!\n");
2688
2689         /* Train 2 */
2690         reg = FDI_TX_CTL(pipe);
2691         temp = I915_READ(reg);
2692         temp &= ~FDI_LINK_TRAIN_NONE;
2693         temp |= FDI_LINK_TRAIN_PATTERN_2;
2694         if (IS_GEN6(dev)) {
2695                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2696                 /* SNB-B */
2697                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
2698         }
2699         I915_WRITE(reg, temp);
2700
2701         reg = FDI_RX_CTL(pipe);
2702         temp = I915_READ(reg);
2703         if (HAS_PCH_CPT(dev)) {
2704                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2705                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
2706         } else {
2707                 temp &= ~FDI_LINK_TRAIN_NONE;
2708                 temp |= FDI_LINK_TRAIN_PATTERN_2;
2709         }
2710         I915_WRITE(reg, temp);
2711
2712         POSTING_READ(reg);
2713         udelay(150);
2714
2715         for (i = 0; i < 4; i++) {
2716                 reg = FDI_TX_CTL(pipe);
2717                 temp = I915_READ(reg);
2718                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2719                 temp |= snb_b_fdi_train_param[i];
2720                 I915_WRITE(reg, temp);
2721
2722                 POSTING_READ(reg);
2723                 udelay(500);
2724
2725                 for (retry = 0; retry < 5; retry++) {
2726                         reg = FDI_RX_IIR(pipe);
2727                         temp = I915_READ(reg);
2728                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2729                         if (temp & FDI_RX_SYMBOL_LOCK) {
2730                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
2731                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
2732                                 break;
2733                         }
2734                         udelay(50);
2735                 }
2736                 if (retry < 5)
2737                         break;
2738         }
2739         if (i == 4)
2740                 DRM_ERROR("FDI train 2 fail!\n");
2741
2742         DRM_DEBUG_KMS("FDI train done.\n");
2743 }
2744
2745 /* Manual link training for Ivy Bridge A0 parts */
2746 static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
2747 {
2748         struct drm_device *dev = crtc->dev;
2749         struct drm_i915_private *dev_priv = dev->dev_private;
2750         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2751         int pipe = intel_crtc->pipe;
2752         u32 reg, temp, i;
2753
2754         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
2755            for train result */
2756         reg = FDI_RX_IMR(pipe);
2757         temp = I915_READ(reg);
2758         temp &= ~FDI_RX_SYMBOL_LOCK;
2759         temp &= ~FDI_RX_BIT_LOCK;
2760         I915_WRITE(reg, temp);
2761
2762         POSTING_READ(reg);
2763         udelay(150);
2764
2765         /* enable CPU FDI TX and PCH FDI RX */
2766         reg = FDI_TX_CTL(pipe);
2767         temp = I915_READ(reg);
2768         temp &= ~(7 << 19);
2769         temp |= (intel_crtc->fdi_lanes - 1) << 19;
2770         temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
2771         temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
2772         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2773         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
2774         temp |= FDI_COMPOSITE_SYNC;
2775         I915_WRITE(reg, temp | FDI_TX_ENABLE);
2776
2777         reg = FDI_RX_CTL(pipe);
2778         temp = I915_READ(reg);
2779         temp &= ~FDI_LINK_TRAIN_AUTO;
2780         temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2781         temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
2782         temp |= FDI_COMPOSITE_SYNC;
2783         I915_WRITE(reg, temp | FDI_RX_ENABLE);
2784
2785         POSTING_READ(reg);
2786         udelay(150);
2787
2788         if (HAS_PCH_CPT(dev))
2789                 cpt_phase_pointer_enable(dev, pipe);
2790
2791         for (i = 0; i < 4; i++) {
2792                 reg = FDI_TX_CTL(pipe);
2793                 temp = I915_READ(reg);
2794                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2795                 temp |= snb_b_fdi_train_param[i];
2796                 I915_WRITE(reg, temp);
2797
2798                 POSTING_READ(reg);
2799                 udelay(500);
2800
2801                 reg = FDI_RX_IIR(pipe);
2802                 temp = I915_READ(reg);
2803                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2804
2805                 if (temp & FDI_RX_BIT_LOCK ||
2806                     (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
2807                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
2808                         DRM_DEBUG_KMS("FDI train 1 done.\n");
2809                         break;
2810                 }
2811         }
2812         if (i == 4)
2813                 DRM_ERROR("FDI train 1 fail!\n");
2814
2815         /* Train 2 */
2816         reg = FDI_TX_CTL(pipe);
2817         temp = I915_READ(reg);
2818         temp &= ~FDI_LINK_TRAIN_NONE_IVB;
2819         temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
2820         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2821         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
2822         I915_WRITE(reg, temp);
2823
2824         reg = FDI_RX_CTL(pipe);
2825         temp = I915_READ(reg);
2826         temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2827         temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
2828         I915_WRITE(reg, temp);
2829
2830         POSTING_READ(reg);
2831         udelay(150);
2832
2833         for (i = 0; i < 4; i++) {
2834                 reg = FDI_TX_CTL(pipe);
2835                 temp = I915_READ(reg);
2836                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
2837                 temp |= snb_b_fdi_train_param[i];
2838                 I915_WRITE(reg, temp);
2839
2840                 POSTING_READ(reg);
2841                 udelay(500);
2842
2843                 reg = FDI_RX_IIR(pipe);
2844                 temp = I915_READ(reg);
2845                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
2846
2847                 if (temp & FDI_RX_SYMBOL_LOCK) {
2848                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
2849                         DRM_DEBUG_KMS("FDI train 2 done.\n");
2850                         break;
2851                 }
2852         }
2853         if (i == 4)
2854                 DRM_ERROR("FDI train 2 fail!\n");
2855
2856         DRM_DEBUG_KMS("FDI train done.\n");
2857 }
2858
2859 static void ironlake_fdi_pll_enable(struct drm_crtc *crtc)
2860 {
2861         struct drm_device *dev = crtc->dev;
2862         struct drm_i915_private *dev_priv = dev->dev_private;
2863         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2864         int pipe = intel_crtc->pipe;
2865         u32 reg, temp;
2866
2867         /* Write the TU size bits so error detection works */
2868         I915_WRITE(FDI_RX_TUSIZE1(pipe),
2869                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
2870
2871         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
2872         reg = FDI_RX_CTL(pipe);
2873         temp = I915_READ(reg);
2874         temp &= ~((0x7 << 19) | (0x7 << 16));
2875         temp |= (intel_crtc->fdi_lanes - 1) << 19;
2876         temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
2877         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
2878
2879         POSTING_READ(reg);
2880         udelay(200);
2881
2882         /* Switch from Rawclk to PCDclk */
2883         temp = I915_READ(reg);
2884         I915_WRITE(reg, temp | FDI_PCDCLK);
2885
2886         POSTING_READ(reg);
2887         udelay(200);
2888
2889         /* Enable CPU FDI TX PLL, always on for Ironlake */
2890         reg = FDI_TX_CTL(pipe);
2891         temp = I915_READ(reg);
2892         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
2893                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
2894
2895                 POSTING_READ(reg);
2896                 udelay(100);
2897         }
2898 }
2899
2900 static void cpt_phase_pointer_disable(struct drm_device *dev, int pipe)
2901 {
2902         struct drm_i915_private *dev_priv = dev->dev_private;
2903         u32 flags = I915_READ(SOUTH_CHICKEN1);
2904
2905         flags &= ~(FDI_PHASE_SYNC_EN(pipe));
2906         I915_WRITE(SOUTH_CHICKEN1, flags); /* once to disable... */
2907         flags &= ~(FDI_PHASE_SYNC_OVR(pipe));
2908         I915_WRITE(SOUTH_CHICKEN1, flags); /* then again to lock */
2909         POSTING_READ(SOUTH_CHICKEN1);
2910 }
2911 static void ironlake_fdi_disable(struct drm_crtc *crtc)
2912 {
2913         struct drm_device *dev = crtc->dev;
2914         struct drm_i915_private *dev_priv = dev->dev_private;
2915         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2916         int pipe = intel_crtc->pipe;
2917         u32 reg, temp;
2918
2919         /* disable CPU FDI tx and PCH FDI rx */
2920         reg = FDI_TX_CTL(pipe);
2921         temp = I915_READ(reg);
2922         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
2923         POSTING_READ(reg);
2924
2925         reg = FDI_RX_CTL(pipe);
2926         temp = I915_READ(reg);
2927         temp &= ~(0x7 << 16);
2928         temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
2929         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
2930
2931         POSTING_READ(reg);
2932         udelay(100);
2933
2934         /* Ironlake workaround, disable clock pointer after downing FDI */
2935         if (HAS_PCH_IBX(dev)) {
2936                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
2937                 I915_WRITE(FDI_RX_CHICKEN(pipe),
2938                            I915_READ(FDI_RX_CHICKEN(pipe) &
2939                                      ~FDI_RX_PHASE_SYNC_POINTER_EN));
2940         } else if (HAS_PCH_CPT(dev)) {
2941                 cpt_phase_pointer_disable(dev, pipe);
2942         }
2943
2944         /* still set train pattern 1 */
2945         reg = FDI_TX_CTL(pipe);
2946         temp = I915_READ(reg);
2947         temp &= ~FDI_LINK_TRAIN_NONE;
2948         temp |= FDI_LINK_TRAIN_PATTERN_1;
2949         I915_WRITE(reg, temp);
2950
2951         reg = FDI_RX_CTL(pipe);
2952         temp = I915_READ(reg);
2953         if (HAS_PCH_CPT(dev)) {
2954                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
2955                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
2956         } else {
2957                 temp &= ~FDI_LINK_TRAIN_NONE;
2958                 temp |= FDI_LINK_TRAIN_PATTERN_1;
2959         }
2960         /* BPC in FDI rx is consistent with that in PIPECONF */
2961         temp &= ~(0x07 << 16);
2962         temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11;
2963         I915_WRITE(reg, temp);
2964
2965         POSTING_READ(reg);
2966         udelay(100);
2967 }
2968
2969 /*
2970  * When we disable a pipe, we need to clear any pending scanline wait events
2971  * to avoid hanging the ring, which we assume we are waiting on.
2972  */
2973 static void intel_clear_scanline_wait(struct drm_device *dev)
2974 {
2975         struct drm_i915_private *dev_priv = dev->dev_private;
2976         struct intel_ring_buffer *ring;
2977         u32 tmp;
2978
2979         if (IS_GEN2(dev))
2980                 /* Can't break the hang on i8xx */
2981                 return;
2982
2983         ring = LP_RING(dev_priv);
2984         tmp = I915_READ_CTL(ring);
2985         if (tmp & RING_WAIT)
2986                 I915_WRITE_CTL(ring, tmp);
2987 }
2988
2989 static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
2990 {
2991         struct drm_i915_gem_object *obj;
2992         struct drm_i915_private *dev_priv;
2993
2994         if (crtc->fb == NULL)
2995                 return;
2996
2997         obj = to_intel_framebuffer(crtc->fb)->obj;
2998         dev_priv = crtc->dev->dev_private;
2999         wait_event(dev_priv->pending_flip_queue,
3000                    atomic_read(&obj->pending_flip) == 0);
3001 }
3002
3003 static bool intel_crtc_driving_pch(struct drm_crtc *crtc)
3004 {
3005         struct drm_device *dev = crtc->dev;
3006         struct drm_mode_config *mode_config = &dev->mode_config;
3007         struct intel_encoder *encoder;
3008
3009         /*
3010          * If there's a non-PCH eDP on this crtc, it must be DP_A, and that
3011          * must be driven by its own crtc; no sharing is possible.
3012          */
3013         list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
3014                 if (encoder->base.crtc != crtc)
3015                         continue;
3016
3017                 switch (encoder->type) {
3018                 case INTEL_OUTPUT_EDP:
3019                         if (!intel_encoder_is_pch_edp(&encoder->base))
3020                                 return false;
3021                         continue;
3022                 }
3023         }
3024
3025         return true;
3026 }
3027
3028 /*
3029  * Enable PCH resources required for PCH ports:
3030  *   - PCH PLLs
3031  *   - FDI training & RX/TX
3032  *   - update transcoder timings
3033  *   - DP transcoding bits
3034  *   - transcoder
3035  */
3036 static void ironlake_pch_enable(struct drm_crtc *crtc)
3037 {
3038         struct drm_device *dev = crtc->dev;
3039         struct drm_i915_private *dev_priv = dev->dev_private;
3040         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3041         int pipe = intel_crtc->pipe;
3042         u32 reg, temp, transc_sel;
3043
3044         /* For PCH output, training FDI link */
3045         dev_priv->display.fdi_link_train(crtc);
3046
3047         intel_enable_pch_pll(dev_priv, pipe);
3048
3049         if (HAS_PCH_CPT(dev)) {
3050                 transc_sel = intel_crtc->use_pll_a ? TRANSC_DPLLA_SEL :
3051                         TRANSC_DPLLB_SEL;
3052
3053                 /* Be sure PCH DPLL SEL is set */
3054                 temp = I915_READ(PCH_DPLL_SEL);
3055                 if (pipe == 0) {
3056                         temp &= ~(TRANSA_DPLLB_SEL);
3057                         temp |= (TRANSA_DPLL_ENABLE | TRANSA_DPLLA_SEL);
3058                 } else if (pipe == 1) {
3059                         temp &= ~(TRANSB_DPLLB_SEL);
3060                         temp |= (TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
3061                 } else if (pipe == 2) {
3062                         temp &= ~(TRANSC_DPLLB_SEL);
3063                         temp |= (TRANSC_DPLL_ENABLE | transc_sel);
3064                 }
3065                 I915_WRITE(PCH_DPLL_SEL, temp);
3066         }
3067
3068         /* set transcoder timing, panel must allow it */
3069         assert_panel_unlocked(dev_priv, pipe);
3070         I915_WRITE(TRANS_HTOTAL(pipe), I915_READ(HTOTAL(pipe)));
3071         I915_WRITE(TRANS_HBLANK(pipe), I915_READ(HBLANK(pipe)));
3072         I915_WRITE(TRANS_HSYNC(pipe),  I915_READ(HSYNC(pipe)));
3073
3074         I915_WRITE(TRANS_VTOTAL(pipe), I915_READ(VTOTAL(pipe)));
3075         I915_WRITE(TRANS_VBLANK(pipe), I915_READ(VBLANK(pipe)));
3076         I915_WRITE(TRANS_VSYNC(pipe),  I915_READ(VSYNC(pipe)));
3077         I915_WRITE(TRANS_VSYNCSHIFT(pipe),  I915_READ(VSYNCSHIFT(pipe)));
3078
3079         intel_fdi_normal_train(crtc);
3080
3081         /* For PCH DP, enable TRANS_DP_CTL */
3082         if (HAS_PCH_CPT(dev) &&
3083             (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
3084              intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))) {
3085                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) >> 5;
3086                 reg = TRANS_DP_CTL(pipe);
3087                 temp = I915_READ(reg);
3088                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
3089                           TRANS_DP_SYNC_MASK |
3090                           TRANS_DP_BPC_MASK);
3091                 temp |= (TRANS_DP_OUTPUT_ENABLE |
3092                          TRANS_DP_ENH_FRAMING);
3093                 temp |= bpc << 9; /* same format but at 11:9 */
3094
3095                 if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
3096                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
3097                 if (crtc->mode.flags & DRM_MODE_FLAG_PVSYNC)
3098                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
3099
3100                 switch (intel_trans_dp_port_sel(crtc)) {
3101                 case PCH_DP_B:
3102                         temp |= TRANS_DP_PORT_SEL_B;
3103                         break;
3104                 case PCH_DP_C:
3105                         temp |= TRANS_DP_PORT_SEL_C;
3106                         break;
3107                 case PCH_DP_D:
3108                         temp |= TRANS_DP_PORT_SEL_D;
3109                         break;
3110                 default:
3111                         DRM_DEBUG_KMS("Wrong PCH DP port return. Guess port B\n");
3112                         temp |= TRANS_DP_PORT_SEL_B;
3113                         break;
3114                 }
3115
3116                 I915_WRITE(reg, temp);
3117         }
3118
3119         intel_enable_transcoder(dev_priv, pipe);
3120 }
3121
3122 void intel_cpt_verify_modeset(struct drm_device *dev, int pipe)
3123 {
3124         struct drm_i915_private *dev_priv = dev->dev_private;
3125         int dslreg = PIPEDSL(pipe), tc2reg = TRANS_CHICKEN2(pipe);
3126         u32 temp;
3127
3128         temp = I915_READ(dslreg);
3129         udelay(500);
3130         if (wait_for(I915_READ(dslreg) != temp, 5)) {
3131                 /* Without this, mode sets may fail silently on FDI */
3132                 I915_WRITE(tc2reg, TRANS_AUTOTRAIN_GEN_STALL_DIS);
3133                 udelay(250);
3134                 I915_WRITE(tc2reg, 0);
3135                 if (wait_for(I915_READ(dslreg) != temp, 5))
3136                         DRM_ERROR("mode set failed: pipe %d stuck\n", pipe);
3137         }
3138 }
3139
3140 static void ironlake_crtc_enable(struct drm_crtc *crtc)
3141 {
3142         struct drm_device *dev = crtc->dev;
3143         struct drm_i915_private *dev_priv = dev->dev_private;
3144         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3145         int pipe = intel_crtc->pipe;
3146         int plane = intel_crtc->plane;
3147         u32 temp;
3148         bool is_pch_port;
3149
3150         if (intel_crtc->active)
3151                 return;
3152
3153         intel_crtc->active = true;
3154         intel_update_watermarks(dev);
3155
3156         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
3157                 temp = I915_READ(PCH_LVDS);
3158                 if ((temp & LVDS_PORT_EN) == 0)
3159                         I915_WRITE(PCH_LVDS, temp | LVDS_PORT_EN);
3160         }
3161
3162         is_pch_port = intel_crtc_driving_pch(crtc);
3163
3164         if (is_pch_port)
3165                 ironlake_fdi_pll_enable(crtc);
3166         else
3167                 ironlake_fdi_disable(crtc);
3168
3169         /* Enable panel fitting for LVDS */
3170         if (dev_priv->pch_pf_size &&
3171             (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) || HAS_eDP)) {
3172                 /* Force use of hard-coded filter coefficients
3173                  * as some pre-programmed values are broken,
3174                  * e.g. x201.
3175                  */
3176                 I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
3177                 I915_WRITE(PF_WIN_POS(pipe), dev_priv->pch_pf_pos);
3178                 I915_WRITE(PF_WIN_SZ(pipe), dev_priv->pch_pf_size);
3179         }
3180
3181         /*
3182          * On ILK+ LUT must be loaded before the pipe is running but with
3183          * clocks enabled
3184          */
3185         intel_crtc_load_lut(crtc);
3186
3187         intel_enable_pipe(dev_priv, pipe, is_pch_port);
3188         intel_enable_plane(dev_priv, plane, pipe);
3189
3190         if (is_pch_port)
3191                 ironlake_pch_enable(crtc);
3192
3193         mutex_lock(&dev->struct_mutex);
3194         intel_update_fbc(dev);
3195         mutex_unlock(&dev->struct_mutex);
3196
3197         intel_crtc_update_cursor(crtc, true);
3198 }
3199
3200 static void ironlake_crtc_disable(struct drm_crtc *crtc)
3201 {
3202         struct drm_device *dev = crtc->dev;
3203         struct drm_i915_private *dev_priv = dev->dev_private;
3204         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3205         int pipe = intel_crtc->pipe;
3206         int plane = intel_crtc->plane;
3207         u32 reg, temp;
3208
3209         if (!intel_crtc->active)
3210                 return;
3211
3212         intel_crtc_wait_for_pending_flips(crtc);
3213         drm_vblank_off(dev, pipe);
3214         intel_crtc_update_cursor(crtc, false);
3215
3216         intel_disable_plane(dev_priv, plane, pipe);
3217
3218         if (dev_priv->cfb_plane == plane)
3219                 intel_disable_fbc(dev);
3220
3221         intel_disable_pipe(dev_priv, pipe);
3222
3223         /* Disable PF */
3224         I915_WRITE(PF_CTL(pipe), 0);
3225         I915_WRITE(PF_WIN_SZ(pipe), 0);
3226
3227         ironlake_fdi_disable(crtc);
3228
3229         /* This is a horrible layering violation; we should be doing this in
3230          * the connector/encoder ->prepare instead, but we don't always have
3231          * enough information there about the config to know whether it will
3232          * actually be necessary or just cause undesired flicker.
3233          */
3234         intel_disable_pch_ports(dev_priv, pipe);
3235
3236         intel_disable_transcoder(dev_priv, pipe);
3237
3238         if (HAS_PCH_CPT(dev)) {
3239                 /* disable TRANS_DP_CTL */
3240                 reg = TRANS_DP_CTL(pipe);
3241                 temp = I915_READ(reg);
3242                 temp &= ~(TRANS_DP_OUTPUT_ENABLE | TRANS_DP_PORT_SEL_MASK);
3243                 temp |= TRANS_DP_PORT_SEL_NONE;
3244                 I915_WRITE(reg, temp);
3245
3246                 /* disable DPLL_SEL */
3247                 temp = I915_READ(PCH_DPLL_SEL);
3248                 switch (pipe) {
3249                 case 0:
3250                         temp &= ~(TRANSA_DPLL_ENABLE | TRANSA_DPLLB_SEL);
3251                         break;
3252                 case 1:
3253                         temp &= ~(TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
3254                         break;
3255                 case 2:
3256                         /* C shares PLL A or B */
3257                         temp &= ~(TRANSC_DPLL_ENABLE | TRANSC_DPLLB_SEL);
3258                         break;
3259                 default:
3260                         BUG(); /* wtf */
3261                 }
3262                 I915_WRITE(PCH_DPLL_SEL, temp);
3263         }
3264
3265         /* disable PCH DPLL */
3266         if (!intel_crtc->no_pll)
3267                 intel_disable_pch_pll(dev_priv, pipe);
3268
3269         /* Switch from PCDclk to Rawclk */
3270         reg = FDI_RX_CTL(pipe);
3271         temp = I915_READ(reg);
3272         I915_WRITE(reg, temp & ~FDI_PCDCLK);
3273
3274         /* Disable CPU FDI TX PLL */
3275         reg = FDI_TX_CTL(pipe);
3276         temp = I915_READ(reg);
3277         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
3278
3279         POSTING_READ(reg);
3280         udelay(100);
3281
3282         reg = FDI_RX_CTL(pipe);
3283         temp = I915_READ(reg);
3284         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
3285
3286         /* Wait for the clocks to turn off. */
3287         POSTING_READ(reg);
3288         udelay(100);
3289
3290         intel_crtc->active = false;
3291         intel_update_watermarks(dev);
3292
3293         mutex_lock(&dev->struct_mutex);
3294         intel_update_fbc(dev);
3295         intel_clear_scanline_wait(dev);
3296         mutex_unlock(&dev->struct_mutex);
3297 }
3298
3299 static void ironlake_crtc_dpms(struct drm_crtc *crtc, int mode)
3300 {
3301         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3302         int pipe = intel_crtc->pipe;
3303         int plane = intel_crtc->plane;
3304
3305         /* XXX: When our outputs are all unaware of DPMS modes other than off
3306          * and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC.
3307          */
3308         switch (mode) {
3309         case DRM_MODE_DPMS_ON:
3310         case DRM_MODE_DPMS_STANDBY:
3311         case DRM_MODE_DPMS_SUSPEND:
3312                 DRM_DEBUG_KMS("crtc %d/%d dpms on\n", pipe, plane);
3313                 ironlake_crtc_enable(crtc);
3314                 break;
3315
3316         case DRM_MODE_DPMS_OFF:
3317                 DRM_DEBUG_KMS("crtc %d/%d dpms off\n", pipe, plane);
3318                 ironlake_crtc_disable(crtc);
3319                 break;
3320         }
3321 }
3322
3323 static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable)
3324 {
3325         if (!enable && intel_crtc->overlay) {
3326                 struct drm_device *dev = intel_crtc->base.dev;
3327                 struct drm_i915_private *dev_priv = dev->dev_private;
3328
3329                 mutex_lock(&dev->struct_mutex);
3330                 dev_priv->mm.interruptible = false;
3331                 (void) intel_overlay_switch_off(intel_crtc->overlay);
3332                 dev_priv->mm.interruptible = true;
3333                 mutex_unlock(&dev->struct_mutex);
3334         }
3335
3336         /* Let userspace switch the overlay on again. In most cases userspace
3337          * has to recompute where to put it anyway.
3338          */
3339 }
3340
3341 static void i9xx_crtc_enable(struct drm_crtc *crtc)
3342 {
3343         struct drm_device *dev = crtc->dev;
3344         struct drm_i915_private *dev_priv = dev->dev_private;
3345         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3346         int pipe = intel_crtc->pipe;
3347         int plane = intel_crtc->plane;
3348
3349         if (intel_crtc->active)
3350                 return;
3351
3352         intel_crtc->active = true;
3353         intel_update_watermarks(dev);
3354
3355         intel_enable_pll(dev_priv, pipe);
3356         intel_enable_pipe(dev_priv, pipe, false);
3357         intel_enable_plane(dev_priv, plane, pipe);
3358
3359         intel_crtc_load_lut(crtc);
3360         intel_update_fbc(dev);
3361
3362         /* Give the overlay scaler a chance to enable if it's on this pipe */
3363         intel_crtc_dpms_overlay(intel_crtc, true);
3364         intel_crtc_update_cursor(crtc, true);
3365 }
3366
3367 static void i9xx_crtc_disable(struct drm_crtc *crtc)
3368 {
3369         struct drm_device *dev = crtc->dev;
3370         struct drm_i915_private *dev_priv = dev->dev_private;
3371         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3372         int pipe = intel_crtc->pipe;
3373         int plane = intel_crtc->plane;
3374
3375         if (!intel_crtc->active)
3376                 return;
3377
3378         /* Give the overlay scaler a chance to disable if it's on this pipe */
3379         intel_crtc_wait_for_pending_flips(crtc);
3380         drm_vblank_off(dev, pipe);
3381         intel_crtc_dpms_overlay(intel_crtc, false);
3382         intel_crtc_update_cursor(crtc, false);
3383
3384         if (dev_priv->cfb_plane == plane)
3385                 intel_disable_fbc(dev);
3386
3387         intel_disable_plane(dev_priv, plane, pipe);
3388         intel_disable_pipe(dev_priv, pipe);
3389         intel_disable_pll(dev_priv, pipe);
3390
3391         intel_crtc->active = false;
3392         intel_update_fbc(dev);
3393         intel_update_watermarks(dev);
3394         intel_clear_scanline_wait(dev);
3395 }
3396
3397 static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
3398 {
3399         /* XXX: When our outputs are all unaware of DPMS modes other than off
3400          * and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC.
3401          */
3402         switch (mode) {
3403         case DRM_MODE_DPMS_ON:
3404         case DRM_MODE_DPMS_STANDBY:
3405         case DRM_MODE_DPMS_SUSPEND:
3406                 i9xx_crtc_enable(crtc);
3407                 break;
3408         case DRM_MODE_DPMS_OFF:
3409                 i9xx_crtc_disable(crtc);
3410                 break;
3411         }
3412 }
3413
3414 /**
3415  * Sets the power management mode of the pipe and plane.
3416  */
3417 static void intel_crtc_dpms(struct drm_crtc *crtc, int mode)
3418 {
3419         struct drm_device *dev = crtc->dev;
3420         struct drm_i915_private *dev_priv = dev->dev_private;
3421         struct drm_i915_master_private *master_priv;
3422         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3423         int pipe = intel_crtc->pipe;
3424         bool enabled;
3425
3426         if (intel_crtc->dpms_mode == mode)
3427                 return;
3428
3429         intel_crtc->dpms_mode = mode;
3430
3431         dev_priv->display.dpms(crtc, mode);
3432
3433         if (!dev->primary->master)
3434                 return;
3435
3436         master_priv = dev->primary->master->driver_priv;
3437         if (!master_priv->sarea_priv)
3438                 return;
3439
3440         enabled = crtc->enabled && mode != DRM_MODE_DPMS_OFF;
3441
3442         switch (pipe) {
3443         case 0:
3444                 master_priv->sarea_priv->pipeA_w = enabled ? crtc->mode.hdisplay : 0;
3445                 master_priv->sarea_priv->pipeA_h = enabled ? crtc->mode.vdisplay : 0;
3446                 break;
3447         case 1:
3448                 master_priv->sarea_priv->pipeB_w = enabled ? crtc->mode.hdisplay : 0;
3449                 master_priv->sarea_priv->pipeB_h = enabled ? crtc->mode.vdisplay : 0;
3450                 break;
3451         default:
3452                 DRM_ERROR("Can't update pipe %c in SAREA\n", pipe_name(pipe));
3453                 break;
3454         }
3455 }
3456
3457 static void intel_crtc_disable(struct drm_crtc *crtc)
3458 {
3459         struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
3460         struct drm_device *dev = crtc->dev;
3461
3462         crtc_funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
3463         assert_plane_disabled(dev->dev_private, to_intel_crtc(crtc)->plane);
3464         assert_pipe_disabled(dev->dev_private, to_intel_crtc(crtc)->pipe);
3465
3466         if (crtc->fb) {
3467                 mutex_lock(&dev->struct_mutex);
3468                 intel_unpin_fb_obj(to_intel_framebuffer(crtc->fb)->obj);
3469                 mutex_unlock(&dev->struct_mutex);
3470         }
3471 }
3472
3473 /* Prepare for a mode set.
3474  *
3475  * Note we could be a lot smarter here.  We need to figure out which outputs
3476  * will be enabled, which disabled (in short, how the config will changes)
3477  * and perform the minimum necessary steps to accomplish that, e.g. updating
3478  * watermarks, FBC configuration, making sure PLLs are programmed correctly,
3479  * panel fitting is in the proper state, etc.
3480  */
3481 static void i9xx_crtc_prepare(struct drm_crtc *crtc)
3482 {
3483         i9xx_crtc_disable(crtc);
3484 }
3485
3486 static void i9xx_crtc_commit(struct drm_crtc *crtc)
3487 {
3488         i9xx_crtc_enable(crtc);
3489 }
3490
3491 static void ironlake_crtc_prepare(struct drm_crtc *crtc)
3492 {
3493         ironlake_crtc_disable(crtc);
3494 }
3495
3496 static void ironlake_crtc_commit(struct drm_crtc *crtc)
3497 {
3498         ironlake_crtc_enable(crtc);
3499 }
3500
3501 void intel_encoder_prepare(struct drm_encoder *encoder)
3502 {
3503         struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
3504         /* lvds has its own version of prepare see intel_lvds_prepare */
3505         encoder_funcs->dpms(encoder, DRM_MODE_DPMS_OFF);
3506 }
3507
3508 void intel_encoder_commit(struct drm_encoder *encoder)
3509 {
3510         struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
3511         struct drm_device *dev = encoder->dev;
3512         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
3513         struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
3514
3515         /* lvds has its own version of commit see intel_lvds_commit */
3516         encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON);
3517
3518         if (HAS_PCH_CPT(dev))
3519                 intel_cpt_verify_modeset(dev, intel_crtc->pipe);
3520 }
3521
3522 void intel_encoder_destroy(struct drm_encoder *encoder)
3523 {
3524         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
3525
3526         drm_encoder_cleanup(encoder);
3527         kfree(intel_encoder);
3528 }
3529
3530 static bool intel_crtc_mode_fixup(struct drm_crtc *crtc,
3531                                   struct drm_display_mode *mode,
3532                                   struct drm_display_mode *adjusted_mode)
3533 {
3534         struct drm_device *dev = crtc->dev;
3535
3536         if (HAS_PCH_SPLIT(dev)) {
3537                 /* FDI link clock is fixed at 2.7G */
3538                 if (mode->clock * 3 > IRONLAKE_FDI_FREQ * 4)
3539                         return false;
3540         }
3541
3542         /* All interlaced capable intel hw wants timings in frames. */
3543         drm_mode_set_crtcinfo(adjusted_mode, 0);
3544
3545         return true;
3546 }
3547
3548 static int valleyview_get_display_clock_speed(struct drm_device *dev)
3549 {
3550         return 400000; /* FIXME */
3551 }
3552
3553 static int i945_get_display_clock_speed(struct drm_device *dev)
3554 {
3555         return 400000;
3556 }
3557
3558 static int i915_get_display_clock_speed(struct drm_device *dev)
3559 {
3560         return 333000;
3561 }
3562
3563 static int i9xx_misc_get_display_clock_speed(struct drm_device *dev)
3564 {
3565         return 200000;
3566 }
3567
3568 static int i915gm_get_display_clock_speed(struct drm_device *dev)
3569 {
3570         u16 gcfgc = 0;
3571
3572         pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
3573
3574         if (gcfgc & GC_LOW_FREQUENCY_ENABLE)
3575                 return 133000;
3576         else {
3577                 switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
3578                 case GC_DISPLAY_CLOCK_333_MHZ:
3579                         return 333000;
3580                 default:
3581                 case GC_DISPLAY_CLOCK_190_200_MHZ:
3582                         return 190000;
3583                 }
3584         }
3585 }
3586
3587 static int i865_get_display_clock_speed(struct drm_device *dev)
3588 {
3589         return 266000;
3590 }
3591
3592 static int i855_get_display_clock_speed(struct drm_device *dev)
3593 {
3594         u16 hpllcc = 0;
3595         /* Assume that the hardware is in the high speed state.  This
3596          * should be the default.
3597          */
3598         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
3599         case GC_CLOCK_133_200:
3600         case GC_CLOCK_100_200:
3601                 return 200000;
3602         case GC_CLOCK_166_250:
3603                 return 250000;
3604         case GC_CLOCK_100_133:
3605                 return 133000;
3606         }
3607
3608         /* Shouldn't happen */
3609         return 0;
3610 }
3611
3612 static int i830_get_display_clock_speed(struct drm_device *dev)
3613 {
3614         return 133000;
3615 }
3616
3617 struct fdi_m_n {
3618         u32        tu;
3619         u32        gmch_m;
3620         u32        gmch_n;
3621         u32        link_m;
3622         u32        link_n;
3623 };
3624
3625 static void
3626 fdi_reduce_ratio(u32 *num, u32 *den)
3627 {
3628         while (*num > 0xffffff || *den > 0xffffff) {
3629                 *num >>= 1;
3630                 *den >>= 1;
3631         }
3632 }
3633
3634 static void
3635 ironlake_compute_m_n(int bits_per_pixel, int nlanes, int pixel_clock,
3636                      int link_clock, struct fdi_m_n *m_n)
3637 {
3638         m_n->tu = 64; /* default size */
3639
3640         /* BUG_ON(pixel_clock > INT_MAX / 36); */
3641         m_n->gmch_m = bits_per_pixel * pixel_clock;
3642         m_n->gmch_n = link_clock * nlanes * 8;
3643         fdi_reduce_ratio(&m_n->gmch_m, &m_n->gmch_n);
3644
3645         m_n->link_m = pixel_clock;
3646         m_n->link_n = link_clock;
3647         fdi_reduce_ratio(&m_n->link_m, &m_n->link_n);
3648 }
3649
3650
3651 struct intel_watermark_params {
3652         unsigned long fifo_size;
3653         unsigned long max_wm;
3654         unsigned long default_wm;
3655         unsigned long guard_size;
3656         unsigned long cacheline_size;
3657 };
3658
3659 /* Pineview has different values for various configs */
3660 static const struct intel_watermark_params pineview_display_wm = {
3661         PINEVIEW_DISPLAY_FIFO,
3662         PINEVIEW_MAX_WM,
3663         PINEVIEW_DFT_WM,
3664         PINEVIEW_GUARD_WM,
3665         PINEVIEW_FIFO_LINE_SIZE
3666 };
3667 static const struct intel_watermark_params pineview_display_hplloff_wm = {
3668         PINEVIEW_DISPLAY_FIFO,
3669         PINEVIEW_MAX_WM,
3670         PINEVIEW_DFT_HPLLOFF_WM,
3671         PINEVIEW_GUARD_WM,
3672         PINEVIEW_FIFO_LINE_SIZE
3673 };
3674 static const struct intel_watermark_params pineview_cursor_wm = {
3675         PINEVIEW_CURSOR_FIFO,
3676         PINEVIEW_CURSOR_MAX_WM,
3677         PINEVIEW_CURSOR_DFT_WM,
3678         PINEVIEW_CURSOR_GUARD_WM,
3679         PINEVIEW_FIFO_LINE_SIZE,
3680 };
3681 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
3682         PINEVIEW_CURSOR_FIFO,
3683         PINEVIEW_CURSOR_MAX_WM,
3684         PINEVIEW_CURSOR_DFT_WM,
3685         PINEVIEW_CURSOR_GUARD_WM,
3686         PINEVIEW_FIFO_LINE_SIZE
3687 };
3688 static const struct intel_watermark_params g4x_wm_info = {
3689         G4X_FIFO_SIZE,
3690         G4X_MAX_WM,
3691         G4X_MAX_WM,
3692         2,
3693         G4X_FIFO_LINE_SIZE,
3694 };
3695 static const struct intel_watermark_params g4x_cursor_wm_info = {
3696         I965_CURSOR_FIFO,
3697         I965_CURSOR_MAX_WM,
3698         I965_CURSOR_DFT_WM,
3699         2,
3700         G4X_FIFO_LINE_SIZE,
3701 };
3702 static const struct intel_watermark_params valleyview_wm_info = {
3703         VALLEYVIEW_FIFO_SIZE,
3704         VALLEYVIEW_MAX_WM,
3705         VALLEYVIEW_MAX_WM,
3706         2,
3707         G4X_FIFO_LINE_SIZE,
3708 };
3709 static const struct intel_watermark_params valleyview_cursor_wm_info = {
3710         I965_CURSOR_FIFO,
3711         VALLEYVIEW_CURSOR_MAX_WM,
3712         I965_CURSOR_DFT_WM,
3713         2,
3714         G4X_FIFO_LINE_SIZE,
3715 };
3716 static const struct intel_watermark_params i965_cursor_wm_info = {
3717         I965_CURSOR_FIFO,
3718         I965_CURSOR_MAX_WM,
3719         I965_CURSOR_DFT_WM,
3720         2,
3721         I915_FIFO_LINE_SIZE,
3722 };
3723 static const struct intel_watermark_params i945_wm_info = {
3724         I945_FIFO_SIZE,
3725         I915_MAX_WM,
3726         1,
3727         2,
3728         I915_FIFO_LINE_SIZE
3729 };
3730 static const struct intel_watermark_params i915_wm_info = {
3731         I915_FIFO_SIZE,
3732         I915_MAX_WM,
3733         1,
3734         2,
3735         I915_FIFO_LINE_SIZE
3736 };
3737 static const struct intel_watermark_params i855_wm_info = {
3738         I855GM_FIFO_SIZE,
3739         I915_MAX_WM,
3740         1,
3741         2,
3742         I830_FIFO_LINE_SIZE
3743 };
3744 static const struct intel_watermark_params i830_wm_info = {
3745         I830_FIFO_SIZE,
3746         I915_MAX_WM,
3747         1,
3748         2,
3749         I830_FIFO_LINE_SIZE
3750 };
3751
3752 static const struct intel_watermark_params ironlake_display_wm_info = {
3753         ILK_DISPLAY_FIFO,
3754         ILK_DISPLAY_MAXWM,
3755         ILK_DISPLAY_DFTWM,
3756         2,
3757         ILK_FIFO_LINE_SIZE
3758 };
3759 static const struct intel_watermark_params ironlake_cursor_wm_info = {
3760         ILK_CURSOR_FIFO,
3761         ILK_CURSOR_MAXWM,
3762         ILK_CURSOR_DFTWM,
3763         2,
3764         ILK_FIFO_LINE_SIZE
3765 };
3766 static const struct intel_watermark_params ironlake_display_srwm_info = {
3767         ILK_DISPLAY_SR_FIFO,
3768         ILK_DISPLAY_MAX_SRWM,
3769         ILK_DISPLAY_DFT_SRWM,
3770         2,
3771         ILK_FIFO_LINE_SIZE
3772 };
3773 static const struct intel_watermark_params ironlake_cursor_srwm_info = {
3774         ILK_CURSOR_SR_FIFO,
3775         ILK_CURSOR_MAX_SRWM,
3776         ILK_CURSOR_DFT_SRWM,
3777         2,
3778         ILK_FIFO_LINE_SIZE
3779 };
3780
3781 static const struct intel_watermark_params sandybridge_display_wm_info = {
3782         SNB_DISPLAY_FIFO,
3783         SNB_DISPLAY_MAXWM,
3784         SNB_DISPLAY_DFTWM,
3785         2,
3786         SNB_FIFO_LINE_SIZE
3787 };
3788 static const struct intel_watermark_params sandybridge_cursor_wm_info = {
3789         SNB_CURSOR_FIFO,
3790         SNB_CURSOR_MAXWM,
3791         SNB_CURSOR_DFTWM,
3792         2,
3793         SNB_FIFO_LINE_SIZE
3794 };
3795 static const struct intel_watermark_params sandybridge_display_srwm_info = {
3796         SNB_DISPLAY_SR_FIFO,
3797         SNB_DISPLAY_MAX_SRWM,
3798         SNB_DISPLAY_DFT_SRWM,
3799         2,
3800         SNB_FIFO_LINE_SIZE
3801 };
3802 static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
3803         SNB_CURSOR_SR_FIFO,
3804         SNB_CURSOR_MAX_SRWM,
3805         SNB_CURSOR_DFT_SRWM,
3806         2,
3807         SNB_FIFO_LINE_SIZE
3808 };
3809
3810
3811 /**
3812  * intel_calculate_wm - calculate watermark level
3813  * @clock_in_khz: pixel clock
3814  * @wm: chip FIFO params
3815  * @pixel_size: display pixel size
3816  * @latency_ns: memory latency for the platform
3817  *
3818  * Calculate the watermark level (the level at which the display plane will
3819  * start fetching from memory again).  Each chip has a different display
3820  * FIFO size and allocation, so the caller needs to figure that out and pass
3821  * in the correct intel_watermark_params structure.
3822  *
3823  * As the pixel clock runs, the FIFO will be drained at a rate that depends
3824  * on the pixel size.  When it reaches the watermark level, it'll start
3825  * fetching FIFO line sized based chunks from memory until the FIFO fills
3826  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
3827  * will occur, and a display engine hang could result.
3828  */
3829 static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
3830                                         const struct intel_watermark_params *wm,
3831                                         int fifo_size,
3832                                         int pixel_size,
3833                                         unsigned long latency_ns)
3834 {
3835         long entries_required, wm_size;
3836
3837         /*
3838          * Note: we need to make sure we don't overflow for various clock &
3839          * latency values.
3840          * clocks go from a few thousand to several hundred thousand.
3841          * latency is usually a few thousand
3842          */
3843         entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
3844                 1000;
3845         entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
3846
3847         DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
3848
3849         wm_size = fifo_size - (entries_required + wm->guard_size);
3850
3851         DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
3852
3853         /* Don't promote wm_size to unsigned... */
3854         if (wm_size > (long)wm->max_wm)
3855                 wm_size = wm->max_wm;
3856         if (wm_size <= 0)
3857                 wm_size = wm->default_wm;
3858         return wm_size;
3859 }
3860
3861 struct cxsr_latency {
3862         int is_desktop;
3863         int is_ddr3;
3864         unsigned long fsb_freq;
3865         unsigned long mem_freq;
3866         unsigned long display_sr;
3867         unsigned long display_hpll_disable;
3868         unsigned long cursor_sr;
3869         unsigned long cursor_hpll_disable;
3870 };
3871
3872 static const struct cxsr_latency cxsr_latency_table[] = {
3873         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
3874         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
3875         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
3876         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
3877         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
3878
3879         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
3880         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
3881         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
3882         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
3883         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
3884
3885         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
3886         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
3887         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
3888         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
3889         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
3890
3891         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
3892         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
3893         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
3894         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
3895         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
3896
3897         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
3898         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
3899         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
3900         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
3901         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
3902
3903         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
3904         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
3905         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
3906         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
3907         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
3908 };
3909
3910 static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
3911                                                          int is_ddr3,
3912                                                          int fsb,
3913                                                          int mem)
3914 {
3915         const struct cxsr_latency *latency;
3916         int i;
3917
3918         if (fsb == 0 || mem == 0)
3919                 return NULL;
3920
3921         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
3922                 latency = &cxsr_latency_table[i];
3923                 if (is_desktop == latency->is_desktop &&
3924                     is_ddr3 == latency->is_ddr3 &&
3925                     fsb == latency->fsb_freq && mem == latency->mem_freq)
3926                         return latency;
3927         }
3928
3929         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
3930
3931         return NULL;
3932 }
3933
3934 static void pineview_disable_cxsr(struct drm_device *dev)
3935 {
3936         struct drm_i915_private *dev_priv = dev->dev_private;
3937
3938         /* deactivate cxsr */
3939         I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
3940 }
3941
3942 /*
3943  * Latency for FIFO fetches is dependent on several factors:
3944  *   - memory configuration (speed, channels)
3945  *   - chipset
3946  *   - current MCH state
3947  * It can be fairly high in some situations, so here we assume a fairly
3948  * pessimal value.  It's a tradeoff between extra memory fetches (if we
3949  * set this value too high, the FIFO will fetch frequently to stay full)
3950  * and power consumption (set it too low to save power and we might see
3951  * FIFO underruns and display "flicker").
3952  *
3953  * A value of 5us seems to be a good balance; safe for very low end
3954  * platforms but not overly aggressive on lower latency configs.
3955  */
3956 static const int latency_ns = 5000;
3957
3958 static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
3959 {
3960         struct drm_i915_private *dev_priv = dev->dev_private;
3961         uint32_t dsparb = I915_READ(DSPARB);
3962         int size;
3963
3964         size = dsparb & 0x7f;
3965         if (plane)
3966                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
3967
3968         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
3969                       plane ? "B" : "A", size);
3970
3971         return size;
3972 }
3973
3974 static int i85x_get_fifo_size(struct drm_device *dev, int plane)
3975 {
3976         struct drm_i915_private *dev_priv = dev->dev_private;
3977         uint32_t dsparb = I915_READ(DSPARB);
3978         int size;
3979
3980         size = dsparb & 0x1ff;
3981         if (plane)
3982                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
3983         size >>= 1; /* Convert to cachelines */
3984
3985         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
3986                       plane ? "B" : "A", size);
3987
3988         return size;
3989 }
3990
3991 static int i845_get_fifo_size(struct drm_device *dev, int plane)
3992 {
3993         struct drm_i915_private *dev_priv = dev->dev_private;
3994         uint32_t dsparb = I915_READ(DSPARB);
3995         int size;
3996
3997         size = dsparb & 0x7f;
3998         size >>= 2; /* Convert to cachelines */
3999
4000         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
4001                       plane ? "B" : "A",
4002                       size);
4003
4004         return size;
4005 }
4006
4007 static int i830_get_fifo_size(struct drm_device *dev, int plane)
4008 {
4009         struct drm_i915_private *dev_priv = dev->dev_private;
4010         uint32_t dsparb = I915_READ(DSPARB);
4011         int size;
4012
4013         size = dsparb & 0x7f;
4014         size >>= 1; /* Convert to cachelines */
4015
4016         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
4017                       plane ? "B" : "A", size);
4018
4019         return size;
4020 }
4021
4022 static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
4023 {
4024         struct drm_crtc *crtc, *enabled = NULL;
4025
4026         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
4027                 if (crtc->enabled && crtc->fb) {
4028                         if (enabled)
4029                                 return NULL;
4030                         enabled = crtc;
4031                 }
4032         }
4033
4034         return enabled;
4035 }
4036
4037 static void pineview_update_wm(struct drm_device *dev)
4038 {
4039         struct drm_i915_private *dev_priv = dev->dev_private;
4040         struct drm_crtc *crtc;
4041         const struct cxsr_latency *latency;
4042         u32 reg;
4043         unsigned long wm;
4044
4045         latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
4046                                          dev_priv->fsb_freq, dev_priv->mem_freq);
4047         if (!latency) {
4048                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
4049                 pineview_disable_cxsr(dev);
4050                 return;
4051         }
4052
4053         crtc = single_enabled_crtc(dev);
4054         if (crtc) {
4055                 int clock = crtc->mode.clock;
4056                 int pixel_size = crtc->fb->bits_per_pixel / 8;
4057
4058                 /* Display SR */
4059                 wm = intel_calculate_wm(clock, &pineview_display_wm,
4060                                         pineview_display_wm.fifo_size,
4061                                         pixel_size, latency->display_sr);
4062                 reg = I915_READ(DSPFW1);
4063                 reg &= ~DSPFW_SR_MASK;
4064                 reg |= wm << DSPFW_SR_SHIFT;
4065                 I915_WRITE(DSPFW1, reg);
4066                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
4067
4068                 /* cursor SR */
4069                 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
4070                                         pineview_display_wm.fifo_size,
4071                                         pixel_size, latency->cursor_sr);
4072                 reg = I915_READ(DSPFW3);
4073                 reg &= ~DSPFW_CURSOR_SR_MASK;
4074                 reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
4075                 I915_WRITE(DSPFW3, reg);
4076
4077                 /* Display HPLL off SR */
4078                 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
4079                                         pineview_display_hplloff_wm.fifo_size,
4080                                         pixel_size, latency->display_hpll_disable);
4081                 reg = I915_READ(DSPFW3);
4082                 reg &= ~DSPFW_HPLL_SR_MASK;
4083                 reg |= wm & DSPFW_HPLL_SR_MASK;
4084                 I915_WRITE(DSPFW3, reg);
4085
4086                 /* cursor HPLL off SR */
4087                 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
4088                                         pineview_display_hplloff_wm.fifo_size,
4089                                         pixel_size, latency->cursor_hpll_disable);
4090                 reg = I915_READ(DSPFW3);
4091                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
4092                 reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
4093                 I915_WRITE(DSPFW3, reg);
4094                 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
4095
4096                 /* activate cxsr */
4097                 I915_WRITE(DSPFW3,
4098                            I915_READ(DSPFW3) | PINEVIEW_SELF_REFRESH_EN);
4099                 DRM_DEBUG_KMS("Self-refresh is enabled\n");
4100         } else {
4101                 pineview_disable_cxsr(dev);
4102                 DRM_DEBUG_KMS("Self-refresh is disabled\n");
4103         }
4104 }
4105
4106 static bool g4x_compute_wm0(struct drm_device *dev,
4107                             int plane,
4108                             const struct intel_watermark_params *display,
4109                             int display_latency_ns,
4110                             const struct intel_watermark_params *cursor,
4111                             int cursor_latency_ns,
4112                             int *plane_wm,
4113                             int *cursor_wm)
4114 {
4115         struct drm_crtc *crtc;
4116         int htotal, hdisplay, clock, pixel_size;
4117         int line_time_us, line_count;
4118         int entries, tlb_miss;
4119
4120         crtc = intel_get_crtc_for_plane(dev, plane);
4121         if (crtc->fb == NULL || !crtc->enabled) {
4122                 *cursor_wm = cursor->guard_size;
4123                 *plane_wm = display->guard_size;
4124                 return false;
4125         }
4126
4127         htotal = crtc->mode.htotal;
4128         hdisplay = crtc->mode.hdisplay;
4129         clock = crtc->mode.clock;
4130         pixel_size = crtc->fb->bits_per_pixel / 8;
4131
4132         /* Use the small buffer method to calculate plane watermark */
4133         entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
4134         tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
4135         if (tlb_miss > 0)
4136                 entries += tlb_miss;
4137         entries = DIV_ROUND_UP(entries, display->cacheline_size);
4138         *plane_wm = entries + display->guard_size;
4139         if (*plane_wm > (int)display->max_wm)
4140                 *plane_wm = display->max_wm;
4141
4142         /* Use the large buffer method to calculate cursor watermark */
4143         line_time_us = ((htotal * 1000) / clock);
4144         line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
4145         entries = line_count * 64 * pixel_size;
4146         tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
4147         if (tlb_miss > 0)
4148                 entries += tlb_miss;
4149         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
4150         *cursor_wm = entries + cursor->guard_size;
4151         if (*cursor_wm > (int)cursor->max_wm)
4152                 *cursor_wm = (int)cursor->max_wm;
4153
4154         return true;
4155 }
4156
4157 /*
4158  * Check the wm result.
4159  *
4160  * If any calculated watermark values is larger than the maximum value that
4161  * can be programmed into the associated watermark register, that watermark
4162  * must be disabled.
4163  */
4164 static bool g4x_check_srwm(struct drm_device *dev,
4165                            int display_wm, int cursor_wm,
4166                            const struct intel_watermark_params *display,
4167                            const struct intel_watermark_params *cursor)
4168 {
4169         DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
4170                       display_wm, cursor_wm);
4171
4172         if (display_wm > display->max_wm) {
4173                 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
4174                               display_wm, display->max_wm);
4175                 return false;
4176         }
4177
4178         if (cursor_wm > cursor->max_wm) {
4179                 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
4180                               cursor_wm, cursor->max_wm);
4181                 return false;
4182         }
4183
4184         if (!(display_wm || cursor_wm)) {
4185                 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
4186                 return false;
4187         }
4188
4189         return true;
4190 }
4191
4192 static bool g4x_compute_srwm(struct drm_device *dev,
4193                              int plane,
4194                              int latency_ns,
4195                              const struct intel_watermark_params *display,
4196                              const struct intel_watermark_params *cursor,
4197                              int *display_wm, int *cursor_wm)
4198 {
4199         struct drm_crtc *crtc;
4200         int hdisplay, htotal, pixel_size, clock;
4201         unsigned long line_time_us;
4202         int line_count, line_size;
4203         int small, large;
4204         int entries;
4205
4206         if (!latency_ns) {
4207                 *display_wm = *cursor_wm = 0;
4208                 return false;
4209         }
4210
4211         crtc = intel_get_crtc_for_plane(dev, plane);
4212         hdisplay = crtc->mode.hdisplay;
4213         htotal = crtc->mode.htotal;
4214         clock = crtc->mode.clock;
4215         pixel_size = crtc->fb->bits_per_pixel / 8;
4216
4217         line_time_us = (htotal * 1000) / clock;
4218         line_count = (latency_ns / line_time_us + 1000) / 1000;
4219         line_size = hdisplay * pixel_size;
4220
4221         /* Use the minimum of the small and large buffer method for primary */
4222         small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
4223         large = line_count * line_size;
4224
4225         entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
4226         *display_wm = entries + display->guard_size;
4227
4228         /* calculate the self-refresh watermark for display cursor */
4229         entries = line_count * pixel_size * 64;
4230         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
4231         *cursor_wm = entries + cursor->guard_size;
4232
4233         return g4x_check_srwm(dev,
4234                               *display_wm, *cursor_wm,
4235                               display, cursor);
4236 }
4237
4238 static bool vlv_compute_drain_latency(struct drm_device *dev,
4239                                      int plane,
4240                                      int *plane_prec_mult,
4241                                      int *plane_dl,
4242                                      int *cursor_prec_mult,
4243                                      int *cursor_dl)
4244 {
4245         struct drm_crtc *crtc;
4246         int clock, pixel_size;
4247         int entries;
4248
4249         crtc = intel_get_crtc_for_plane(dev, plane);
4250         if (crtc->fb == NULL || !crtc->enabled)
4251                 return false;
4252
4253         clock = crtc->mode.clock;       /* VESA DOT Clock */
4254         pixel_size = crtc->fb->bits_per_pixel / 8;      /* BPP */
4255
4256         entries = (clock / 1000) * pixel_size;
4257         *plane_prec_mult = (entries > 256) ?
4258                 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
4259         *plane_dl = (64 * (*plane_prec_mult) * 4) / ((clock / 1000) *
4260                                                      pixel_size);
4261
4262         entries = (clock / 1000) * 4;   /* BPP is always 4 for cursor */
4263         *cursor_prec_mult = (entries > 256) ?
4264                 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
4265         *cursor_dl = (64 * (*cursor_prec_mult) * 4) / ((clock / 1000) * 4);
4266
4267         return true;
4268 }
4269
4270 /*
4271  * Update drain latency registers of memory arbiter
4272  *
4273  * Valleyview SoC has a new memory arbiter and needs drain latency registers
4274  * to be programmed. Each plane has a drain latency multiplier and a drain
4275  * latency value.
4276  */
4277
4278 static void vlv_update_drain_latency(struct drm_device *dev)
4279 {
4280         struct drm_i915_private *dev_priv = dev->dev_private;
4281         int planea_prec, planea_dl, planeb_prec, planeb_dl;
4282         int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
4283         int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
4284                                                         either 16 or 32 */
4285
4286         /* For plane A, Cursor A */
4287         if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
4288                                       &cursor_prec_mult, &cursora_dl)) {
4289                 cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
4290                         DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
4291                 planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
4292                         DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
4293
4294                 I915_WRITE(VLV_DDL1, cursora_prec |
4295                                 (cursora_dl << DDL_CURSORA_SHIFT) |
4296                                 planea_prec | planea_dl);
4297         }
4298
4299         /* For plane B, Cursor B */
4300         if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
4301                                       &cursor_prec_mult, &cursorb_dl)) {
4302                 cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
4303                         DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
4304                 planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
4305                         DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
4306
4307                 I915_WRITE(VLV_DDL2, cursorb_prec |
4308                                 (cursorb_dl << DDL_CURSORB_SHIFT) |
4309                                 planeb_prec | planeb_dl);
4310         }
4311 }
4312
4313 #define single_plane_enabled(mask) is_power_of_2(mask)
4314
4315 static void valleyview_update_wm(struct drm_device *dev)
4316 {
4317         static const int sr_latency_ns = 12000;
4318         struct drm_i915_private *dev_priv = dev->dev_private;
4319         int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
4320         int plane_sr, cursor_sr;
4321         unsigned int enabled = 0;
4322
4323         vlv_update_drain_latency(dev);
4324
4325         if (g4x_compute_wm0(dev, 0,
4326                             &valleyview_wm_info, latency_ns,
4327                             &valleyview_cursor_wm_info, latency_ns,
4328                             &planea_wm, &cursora_wm))
4329                 enabled |= 1;
4330
4331         if (g4x_compute_wm0(dev, 1,
4332                             &valleyview_wm_info, latency_ns,
4333                             &valleyview_cursor_wm_info, latency_ns,
4334                             &planeb_wm, &cursorb_wm))
4335                 enabled |= 2;
4336
4337         plane_sr = cursor_sr = 0;
4338         if (single_plane_enabled(enabled) &&
4339             g4x_compute_srwm(dev, ffs(enabled) - 1,
4340                              sr_latency_ns,
4341                              &valleyview_wm_info,
4342                              &valleyview_cursor_wm_info,
4343                              &plane_sr, &cursor_sr))
4344                 I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
4345         else
4346                 I915_WRITE(FW_BLC_SELF_VLV,
4347                            I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
4348
4349         DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
4350                       planea_wm, cursora_wm,
4351                       planeb_wm, cursorb_wm,
4352                       plane_sr, cursor_sr);
4353
4354         I915_WRITE(DSPFW1,
4355                    (plane_sr << DSPFW_SR_SHIFT) |
4356                    (cursorb_wm << DSPFW_CURSORB_SHIFT) |
4357                    (planeb_wm << DSPFW_PLANEB_SHIFT) |
4358                    planea_wm);
4359         I915_WRITE(DSPFW2,
4360                    (I915_READ(DSPFW2) & DSPFW_CURSORA_MASK) |
4361                    (cursora_wm << DSPFW_CURSORA_SHIFT));
4362         I915_WRITE(DSPFW3,
4363                    (I915_READ(DSPFW3) | (cursor_sr << DSPFW_CURSOR_SR_SHIFT)));
4364 }
4365
4366 static void g4x_update_wm(struct drm_device *dev)
4367 {
4368         static const int sr_latency_ns = 12000;
4369         struct drm_i915_private *dev_priv = dev->dev_private;
4370         int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
4371         int plane_sr, cursor_sr;
4372         unsigned int enabled = 0;
4373
4374         if (g4x_compute_wm0(dev, 0,
4375                             &g4x_wm_info, latency_ns,
4376                             &g4x_cursor_wm_info, latency_ns,
4377                             &planea_wm, &cursora_wm))
4378                 enabled |= 1;
4379
4380         if (g4x_compute_wm0(dev, 1,
4381                             &g4x_wm_info, latency_ns,
4382                             &g4x_cursor_wm_info, latency_ns,
4383                             &planeb_wm, &cursorb_wm))
4384                 enabled |= 2;
4385
4386         plane_sr = cursor_sr = 0;
4387         if (single_plane_enabled(enabled) &&
4388             g4x_compute_srwm(dev, ffs(enabled) - 1,
4389                              sr_latency_ns,
4390                              &g4x_wm_info,
4391                              &g4x_cursor_wm_info,
4392                              &plane_sr, &cursor_sr))
4393                 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
4394         else
4395                 I915_WRITE(FW_BLC_SELF,
4396                            I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
4397
4398         DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
4399                       planea_wm, cursora_wm,
4400                       planeb_wm, cursorb_wm,
4401                       plane_sr, cursor_sr);
4402
4403         I915_WRITE(DSPFW1,
4404                    (plane_sr << DSPFW_SR_SHIFT) |
4405                    (cursorb_wm << DSPFW_CURSORB_SHIFT) |
4406                    (planeb_wm << DSPFW_PLANEB_SHIFT) |
4407                    planea_wm);
4408         I915_WRITE(DSPFW2,
4409                    (I915_READ(DSPFW2) & DSPFW_CURSORA_MASK) |
4410                    (cursora_wm << DSPFW_CURSORA_SHIFT));
4411         /* HPLL off in SR has some issues on G4x... disable it */
4412         I915_WRITE(DSPFW3,
4413                    (I915_READ(DSPFW3) & ~DSPFW_HPLL_SR_EN) |
4414                    (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
4415 }
4416
4417 static void i965_update_wm(struct drm_device *dev)
4418 {
4419         struct drm_i915_private *dev_priv = dev->dev_private;
4420         struct drm_crtc *crtc;
4421         int srwm = 1;
4422         int cursor_sr = 16;
4423
4424         /* Calc sr entries for one plane configs */
4425         crtc = single_enabled_crtc(dev);
4426         if (crtc) {
4427                 /* self-refresh has much higher latency */
4428                 static const int sr_latency_ns = 12000;
4429                 int clock = crtc->mode.clock;
4430                 int htotal = crtc->mode.htotal;
4431                 int hdisplay = crtc->mode.hdisplay;
4432                 int pixel_size = crtc->fb->bits_per_pixel / 8;
4433                 unsigned long line_time_us;
4434                 int entries;
4435
4436                 line_time_us = ((htotal * 1000) / clock);
4437
4438                 /* Use ns/us then divide to preserve precision */
4439                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
4440                         pixel_size * hdisplay;
4441                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
4442                 srwm = I965_FIFO_SIZE - entries;
4443                 if (srwm < 0)
4444                         srwm = 1;
4445                 srwm &= 0x1ff;
4446                 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
4447                               entries, srwm);
4448
4449                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
4450                         pixel_size * 64;
4451                 entries = DIV_ROUND_UP(entries,
4452                                           i965_cursor_wm_info.cacheline_size);
4453                 cursor_sr = i965_cursor_wm_info.fifo_size -
4454                         (entries + i965_cursor_wm_info.guard_size);
4455
4456                 if (cursor_sr > i965_cursor_wm_info.max_wm)
4457                         cursor_sr = i965_cursor_wm_info.max_wm;
4458
4459                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
4460                               "cursor %d\n", srwm, cursor_sr);
4461
4462                 if (IS_CRESTLINE(dev))
4463                         I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
4464         } else {
4465                 /* Turn off self refresh if both pipes are enabled */
4466                 if (IS_CRESTLINE(dev))
4467                         I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
4468                                    & ~FW_BLC_SELF_EN);
4469         }
4470
4471         DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
4472                       srwm);
4473
4474         /* 965 has limitations... */
4475         I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
4476                    (8 << 16) | (8 << 8) | (8 << 0));
4477         I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
4478         /* update cursor SR watermark */
4479         I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
4480 }
4481
4482 static void i9xx_update_wm(struct drm_device *dev)
4483 {
4484         struct drm_i915_private *dev_priv = dev->dev_private;
4485         const struct intel_watermark_params *wm_info;
4486         uint32_t fwater_lo;
4487         uint32_t fwater_hi;
4488         int cwm, srwm = 1;
4489         int fifo_size;
4490         int planea_wm, planeb_wm;
4491         struct drm_crtc *crtc, *enabled = NULL;
4492
4493         if (IS_I945GM(dev))
4494                 wm_info = &i945_wm_info;
4495         else if (!IS_GEN2(dev))
4496                 wm_info = &i915_wm_info;
4497         else
4498                 wm_info = &i855_wm_info;
4499
4500         fifo_size = dev_priv->display.get_fifo_size(dev, 0);
4501         crtc = intel_get_crtc_for_plane(dev, 0);
4502         if (crtc->enabled && crtc->fb) {
4503                 planea_wm = intel_calculate_wm(crtc->mode.clock,
4504                                                wm_info, fifo_size,
4505                                                crtc->fb->bits_per_pixel / 8,
4506                                                latency_ns);
4507                 enabled = crtc;
4508         } else
4509                 planea_wm = fifo_size - wm_info->guard_size;
4510
4511         fifo_size = dev_priv->display.get_fifo_size(dev, 1);
4512         crtc = intel_get_crtc_for_plane(dev, 1);
4513         if (crtc->enabled && crtc->fb) {
4514                 planeb_wm = intel_calculate_wm(crtc->mode.clock,
4515                                                wm_info, fifo_size,
4516                                                crtc->fb->bits_per_pixel / 8,
4517                                                latency_ns);
4518                 if (enabled == NULL)
4519                         enabled = crtc;
4520                 else
4521                         enabled = NULL;
4522         } else
4523                 planeb_wm = fifo_size - wm_info->guard_size;
4524
4525         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
4526
4527         /*
4528          * Overlay gets an aggressive default since video jitter is bad.
4529          */
4530         cwm = 2;
4531
4532         /* Play safe and disable self-refresh before adjusting watermarks. */
4533         if (IS_I945G(dev) || IS_I945GM(dev))
4534                 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | 0);
4535         else if (IS_I915GM(dev))
4536                 I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
4537
4538         /* Calc sr entries for one plane configs */
4539         if (HAS_FW_BLC(dev) && enabled) {
4540                 /* self-refresh has much higher latency */
4541                 static const int sr_latency_ns = 6000;
4542                 int clock = enabled->mode.clock;
4543                 int htotal = enabled->mode.htotal;
4544                 int hdisplay = enabled->mode.hdisplay;
4545                 int pixel_size = enabled->fb->bits_per_pixel / 8;
4546                 unsigned long line_time_us;
4547                 int entries;
4548
4549                 line_time_us = (htotal * 1000) / clock;
4550
4551                 /* Use ns/us then divide to preserve precision */
4552                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
4553                         pixel_size * hdisplay;
4554                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
4555                 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
4556                 srwm = wm_info->fifo_size - entries;
4557                 if (srwm < 0)
4558                         srwm = 1;
4559
4560                 if (IS_I945G(dev) || IS_I945GM(dev))
4561                         I915_WRITE(FW_BLC_SELF,
4562                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
4563                 else if (IS_I915GM(dev))
4564                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
4565         }
4566
4567         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
4568                       planea_wm, planeb_wm, cwm, srwm);
4569
4570         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
4571         fwater_hi = (cwm & 0x1f);
4572
4573         /* Set request length to 8 cachelines per fetch */
4574         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
4575         fwater_hi = fwater_hi | (1 << 8);
4576
4577         I915_WRITE(FW_BLC, fwater_lo);
4578         I915_WRITE(FW_BLC2, fwater_hi);
4579
4580         if (HAS_FW_BLC(dev)) {
4581                 if (enabled) {
4582                         if (IS_I945G(dev) || IS_I945GM(dev))
4583                                 I915_WRITE(FW_BLC_SELF,
4584                                            FW_BLC_SELF_EN_MASK | FW_BLC_SELF_EN);
4585                         else if (IS_I915GM(dev))
4586                                 I915_WRITE(INSTPM, I915_READ(INSTPM) | INSTPM_SELF_EN);
4587                         DRM_DEBUG_KMS("memory self refresh enabled\n");
4588                 } else
4589                         DRM_DEBUG_KMS("memory self refresh disabled\n");
4590         }
4591 }
4592
4593 static void i830_update_wm(struct drm_device *dev)
4594 {
4595         struct drm_i915_private *dev_priv = dev->dev_private;
4596         struct drm_crtc *crtc;
4597         uint32_t fwater_lo;
4598         int planea_wm;
4599
4600         crtc = single_enabled_crtc(dev);
4601         if (crtc == NULL)
4602                 return;
4603
4604         planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
4605                                        dev_priv->display.get_fifo_size(dev, 0),
4606                                        crtc->fb->bits_per_pixel / 8,
4607                                        latency_ns);
4608         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
4609         fwater_lo |= (3<<8) | planea_wm;
4610
4611         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
4612
4613         I915_WRITE(FW_BLC, fwater_lo);
4614 }
4615
4616 #define ILK_LP0_PLANE_LATENCY           700
4617 #define ILK_LP0_CURSOR_LATENCY          1300
4618
4619 /*
4620  * Check the wm result.
4621  *
4622  * If any calculated watermark values is larger than the maximum value that
4623  * can be programmed into the associated watermark register, that watermark
4624  * must be disabled.
4625  */
4626 static bool ironlake_check_srwm(struct drm_device *dev, int level,
4627                                 int fbc_wm, int display_wm, int cursor_wm,
4628                                 const struct intel_watermark_params *display,
4629                                 const struct intel_watermark_params *cursor)
4630 {
4631         struct drm_i915_private *dev_priv = dev->dev_private;
4632
4633         DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
4634                       " cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
4635
4636         if (fbc_wm > SNB_FBC_MAX_SRWM) {
4637                 DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
4638                               fbc_wm, SNB_FBC_MAX_SRWM, level);
4639
4640                 /* fbc has it's own way to disable FBC WM */
4641                 I915_WRITE(DISP_ARB_CTL,
4642                            I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS);
4643                 return false;
4644         }
4645
4646         if (display_wm > display->max_wm) {
4647                 DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
4648                               display_wm, SNB_DISPLAY_MAX_SRWM, level);
4649                 return false;
4650         }
4651
4652         if (cursor_wm > cursor->max_wm) {
4653                 DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
4654                               cursor_wm, SNB_CURSOR_MAX_SRWM, level);
4655                 return false;
4656         }
4657
4658         if (!(fbc_wm || display_wm || cursor_wm)) {
4659                 DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
4660                 return false;
4661         }
4662
4663         return true;
4664 }
4665
4666 /*
4667  * Compute watermark values of WM[1-3],
4668  */
4669 static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
4670                                   int latency_ns,
4671                                   const struct intel_watermark_params *display,
4672                                   const struct intel_watermark_params *cursor,
4673                                   int *fbc_wm, int *display_wm, int *cursor_wm)
4674 {
4675         struct drm_crtc *crtc;
4676         unsigned long line_time_us;
4677         int hdisplay, htotal, pixel_size, clock;
4678         int line_count, line_size;
4679         int small, large;
4680         int entries;
4681
4682         if (!latency_ns) {
4683                 *fbc_wm = *display_wm = *cursor_wm = 0;
4684                 return false;
4685         }
4686
4687         crtc = intel_get_crtc_for_plane(dev, plane);
4688         hdisplay = crtc->mode.hdisplay;
4689         htotal = crtc->mode.htotal;
4690         clock = crtc->mode.clock;
4691         pixel_size = crtc->fb->bits_per_pixel / 8;
4692
4693         line_time_us = (htotal * 1000) / clock;
4694         line_count = (latency_ns / line_time_us + 1000) / 1000;
4695         line_size = hdisplay * pixel_size;
4696
4697         /* Use the minimum of the small and large buffer method for primary */
4698         small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
4699         large = line_count * line_size;
4700
4701         entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
4702         *display_wm = entries + display->guard_size;
4703
4704         /*
4705          * Spec says:
4706          * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
4707          */
4708         *fbc_wm = DIV_ROUND_UP(*display_wm * 64, line_size) + 2;
4709
4710         /* calculate the self-refresh watermark for display cursor */
4711         entries = line_count * pixel_size * 64;
4712         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
4713         *cursor_wm = entries + cursor->guard_size;
4714
4715         return ironlake_check_srwm(dev, level,
4716                                    *fbc_wm, *display_wm, *cursor_wm,
4717                                    display, cursor);
4718 }
4719
4720 static void ironlake_update_wm(struct drm_device *dev)
4721 {
4722         struct drm_i915_private *dev_priv = dev->dev_private;
4723         int fbc_wm, plane_wm, cursor_wm;
4724         unsigned int enabled;
4725
4726         enabled = 0;
4727         if (g4x_compute_wm0(dev, 0,
4728                             &ironlake_display_wm_info,
4729                             ILK_LP0_PLANE_LATENCY,
4730                             &ironlake_cursor_wm_info,
4731                             ILK_LP0_CURSOR_LATENCY,
4732                             &plane_wm, &cursor_wm)) {
4733                 I915_WRITE(WM0_PIPEA_ILK,
4734                            (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
4735                 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
4736                               " plane %d, " "cursor: %d\n",
4737                               plane_wm, cursor_wm);
4738                 enabled |= 1;
4739         }
4740
4741         if (g4x_compute_wm0(dev, 1,
4742                             &ironlake_display_wm_info,
4743                             ILK_LP0_PLANE_LATENCY,
4744                             &ironlake_cursor_wm_info,
4745                             ILK_LP0_CURSOR_LATENCY,
4746                             &plane_wm, &cursor_wm)) {
4747                 I915_WRITE(WM0_PIPEB_ILK,
4748                            (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
4749                 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
4750                               " plane %d, cursor: %d\n",
4751                               plane_wm, cursor_wm);
4752                 enabled |= 2;
4753         }
4754
4755         /*
4756          * Calculate and update the self-refresh watermark only when one
4757          * display plane is used.
4758          */
4759         I915_WRITE(WM3_LP_ILK, 0);
4760         I915_WRITE(WM2_LP_ILK, 0);
4761         I915_WRITE(WM1_LP_ILK, 0);
4762
4763         if (!single_plane_enabled(enabled))
4764                 return;
4765         enabled = ffs(enabled) - 1;
4766
4767         /* WM1 */
4768         if (!ironlake_compute_srwm(dev, 1, enabled,
4769                                    ILK_READ_WM1_LATENCY() * 500,
4770                                    &ironlake_display_srwm_info,
4771                                    &ironlake_cursor_srwm_info,
4772                                    &fbc_wm, &plane_wm, &cursor_wm))
4773                 return;
4774
4775         I915_WRITE(WM1_LP_ILK,
4776                    WM1_LP_SR_EN |
4777                    (ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
4778                    (fbc_wm << WM1_LP_FBC_SHIFT) |
4779                    (plane_wm << WM1_LP_SR_SHIFT) |
4780                    cursor_wm);
4781
4782         /* WM2 */
4783         if (!ironlake_compute_srwm(dev, 2, enabled,
4784                                    ILK_READ_WM2_LATENCY() * 500,
4785                                    &ironlake_display_srwm_info,
4786                                    &ironlake_cursor_srwm_info,
4787                                    &fbc_wm, &plane_wm, &cursor_wm))
4788                 return;
4789
4790         I915_WRITE(WM2_LP_ILK,
4791                    WM2_LP_EN |
4792                    (ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
4793                    (fbc_wm << WM1_LP_FBC_SHIFT) |
4794                    (plane_wm << WM1_LP_SR_SHIFT) |
4795                    cursor_wm);
4796
4797         /*
4798          * WM3 is unsupported on ILK, probably because we don't have latency
4799          * data for that power state
4800          */
4801 }
4802
4803 void sandybridge_update_wm(struct drm_device *dev)
4804 {
4805         struct drm_i915_private *dev_priv = dev->dev_private;
4806         int latency = SNB_READ_WM0_LATENCY() * 100;     /* In unit 0.1us */
4807         u32 val;
4808         int fbc_wm, plane_wm, cursor_wm;
4809         unsigned int enabled;
4810
4811         enabled = 0;
4812         if (g4x_compute_wm0(dev, 0,
4813                             &sandybridge_display_wm_info, latency,
4814                             &sandybridge_cursor_wm_info, latency,
4815                             &plane_wm, &cursor_wm)) {
4816                 val = I915_READ(WM0_PIPEA_ILK);
4817                 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
4818                 I915_WRITE(WM0_PIPEA_ILK, val |
4819                            ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
4820                 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
4821                               " plane %d, " "cursor: %d\n",
4822                               plane_wm, cursor_wm);
4823                 enabled |= 1;
4824         }
4825
4826         if (g4x_compute_wm0(dev, 1,
4827                             &sandybridge_display_wm_info, latency,
4828                             &sandybridge_cursor_wm_info, latency,
4829                             &plane_wm, &cursor_wm)) {
4830                 val = I915_READ(WM0_PIPEB_ILK);
4831                 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
4832                 I915_WRITE(WM0_PIPEB_ILK, val |
4833                            ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
4834                 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
4835                               " plane %d, cursor: %d\n",
4836                               plane_wm, cursor_wm);
4837                 enabled |= 2;
4838         }
4839
4840         /* IVB has 3 pipes */
4841         if (IS_IVYBRIDGE(dev) &&
4842             g4x_compute_wm0(dev, 2,
4843                             &sandybridge_display_wm_info, latency,
4844                             &sandybridge_cursor_wm_info, latency,
4845                             &plane_wm, &cursor_wm)) {
4846                 val = I915_READ(WM0_PIPEC_IVB);
4847                 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
4848                 I915_WRITE(WM0_PIPEC_IVB, val |
4849                            ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
4850                 DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
4851                               " plane %d, cursor: %d\n",
4852                               plane_wm, cursor_wm);
4853                 enabled |= 3;
4854         }
4855
4856         /*
4857          * Calculate and update the self-refresh watermark only when one
4858          * display plane is used.
4859          *
4860          * SNB support 3 levels of watermark.
4861          *
4862          * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
4863          * and disabled in the descending order
4864          *
4865          */
4866         I915_WRITE(WM3_LP_ILK, 0);
4867         I915_WRITE(WM2_LP_ILK, 0);
4868         I915_WRITE(WM1_LP_ILK, 0);
4869
4870         if (!single_plane_enabled(enabled) ||
4871             dev_priv->sprite_scaling_enabled)
4872                 return;
4873         enabled = ffs(enabled) - 1;
4874
4875         /* WM1 */
4876         if (!ironlake_compute_srwm(dev, 1, enabled,
4877                                    SNB_READ_WM1_LATENCY() * 500,
4878                                    &sandybridge_display_srwm_info,
4879                                    &sandybridge_cursor_srwm_info,
4880                                    &fbc_wm, &plane_wm, &cursor_wm))
4881                 return;
4882
4883         I915_WRITE(WM1_LP_ILK,
4884                    WM1_LP_SR_EN |
4885                    (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
4886                    (fbc_wm << WM1_LP_FBC_SHIFT) |
4887                    (plane_wm << WM1_LP_SR_SHIFT) |
4888                    cursor_wm);
4889
4890         /* WM2 */
4891         if (!ironlake_compute_srwm(dev, 2, enabled,
4892                                    SNB_READ_WM2_LATENCY() * 500,
4893                                    &sandybridge_display_srwm_info,
4894                                    &sandybridge_cursor_srwm_info,
4895                                    &fbc_wm, &plane_wm, &cursor_wm))
4896                 return;
4897
4898         I915_WRITE(WM2_LP_ILK,
4899                    WM2_LP_EN |
4900                    (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
4901                    (fbc_wm << WM1_LP_FBC_SHIFT) |
4902                    (plane_wm << WM1_LP_SR_SHIFT) |
4903                    cursor_wm);
4904
4905         /* WM3 */
4906         if (!ironlake_compute_srwm(dev, 3, enabled,
4907                                    SNB_READ_WM3_LATENCY() * 500,
4908                                    &sandybridge_display_srwm_info,
4909                                    &sandybridge_cursor_srwm_info,
4910                                    &fbc_wm, &plane_wm, &cursor_wm))
4911                 return;
4912
4913         I915_WRITE(WM3_LP_ILK,
4914                    WM3_LP_EN |
4915                    (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) |
4916                    (fbc_wm << WM1_LP_FBC_SHIFT) |
4917                    (plane_wm << WM1_LP_SR_SHIFT) |
4918                    cursor_wm);
4919 }
4920
4921 static bool
4922 sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
4923                               uint32_t sprite_width, int pixel_size,
4924                               const struct intel_watermark_params *display,
4925                               int display_latency_ns, int *sprite_wm)
4926 {
4927         struct drm_crtc *crtc;
4928         int clock;
4929         int entries, tlb_miss;
4930
4931         crtc = intel_get_crtc_for_plane(dev, plane);
4932         if (crtc->fb == NULL || !crtc->enabled) {
4933                 *sprite_wm = display->guard_size;
4934                 return false;
4935         }
4936
4937         clock = crtc->mode.clock;
4938
4939         /* Use the small buffer method to calculate the sprite watermark */
4940         entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
4941         tlb_miss = display->fifo_size*display->cacheline_size -
4942                 sprite_width * 8;
4943         if (tlb_miss > 0)
4944                 entries += tlb_miss;
4945         entries = DIV_ROUND_UP(entries, display->cacheline_size);
4946         *sprite_wm = entries + display->guard_size;
4947         if (*sprite_wm > (int)display->max_wm)
4948                 *sprite_wm = display->max_wm;
4949
4950         return true;
4951 }
4952
4953 static bool
4954 sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
4955                                 uint32_t sprite_width, int pixel_size,
4956                                 const struct intel_watermark_params *display,
4957                                 int latency_ns, int *sprite_wm)
4958 {
4959         struct drm_crtc *crtc;
4960         unsigned long line_time_us;
4961         int clock;
4962         int line_count, line_size;
4963         int small, large;
4964         int entries;
4965
4966         if (!latency_ns) {
4967                 *sprite_wm = 0;
4968                 return false;
4969         }
4970
4971         crtc = intel_get_crtc_for_plane(dev, plane);
4972         clock = crtc->mode.clock;
4973
4974         line_time_us = (sprite_width * 1000) / clock;
4975         line_count = (latency_ns / line_time_us + 1000) / 1000;
4976         line_size = sprite_width * pixel_size;
4977
4978         /* Use the minimum of the small and large buffer method for primary */
4979         small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
4980         large = line_count * line_size;
4981
4982         entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
4983         *sprite_wm = entries + display->guard_size;
4984
4985         return *sprite_wm > 0x3ff ? false : true;
4986 }
4987
4988 static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
4989                                          uint32_t sprite_width, int pixel_size)
4990 {
4991         struct drm_i915_private *dev_priv = dev->dev_private;
4992         int latency = SNB_READ_WM0_LATENCY() * 100;     /* In unit 0.1us */
4993         u32 val;
4994         int sprite_wm, reg;
4995         int ret;
4996
4997         switch (pipe) {
4998         case 0:
4999                 reg = WM0_PIPEA_ILK;
5000                 break;
5001         case 1:
5002                 reg = WM0_PIPEB_ILK;
5003                 break;
5004         case 2:
5005                 reg = WM0_PIPEC_IVB;
5006                 break;
5007         default:
5008                 return; /* bad pipe */
5009         }
5010
5011         ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
5012                                             &sandybridge_display_wm_info,
5013                                             latency, &sprite_wm);
5014         if (!ret) {
5015                 DRM_DEBUG_KMS("failed to compute sprite wm for pipe %d\n",
5016                               pipe);
5017                 return;
5018         }
5019
5020         val = I915_READ(reg);
5021         val &= ~WM0_PIPE_SPRITE_MASK;
5022         I915_WRITE(reg, val | (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
5023         DRM_DEBUG_KMS("sprite watermarks For pipe %d - %d\n", pipe, sprite_wm);
5024
5025
5026         ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
5027                                               pixel_size,
5028                                               &sandybridge_display_srwm_info,
5029                                               SNB_READ_WM1_LATENCY() * 500,
5030                                               &sprite_wm);
5031         if (!ret) {
5032                 DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %d\n",
5033                               pipe);
5034                 return;
5035         }
5036         I915_WRITE(WM1S_LP_ILK, sprite_wm);
5037
5038         /* Only IVB has two more LP watermarks for sprite */
5039         if (!IS_IVYBRIDGE(dev))
5040                 return;
5041
5042         ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
5043                                               pixel_size,
5044                                               &sandybridge_display_srwm_info,
5045                                               SNB_READ_WM2_LATENCY() * 500,
5046                                               &sprite_wm);
5047         if (!ret) {
5048                 DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %d\n",
5049                               pipe);
5050                 return;
5051         }
5052         I915_WRITE(WM2S_LP_IVB, sprite_wm);
5053
5054         ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
5055                                               pixel_size,
5056                                               &sandybridge_display_srwm_info,
5057                                               SNB_READ_WM3_LATENCY() * 500,
5058                                               &sprite_wm);
5059         if (!ret) {
5060                 DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %d\n",
5061                               pipe);
5062                 return;
5063         }
5064         I915_WRITE(WM3S_LP_IVB, sprite_wm);
5065 }
5066
5067 /**
5068  * intel_update_watermarks - update FIFO watermark values based on current modes
5069  *
5070  * Calculate watermark values for the various WM regs based on current mode
5071  * and plane configuration.
5072  *
5073  * There are several cases to deal with here:
5074  *   - normal (i.e. non-self-refresh)
5075  *   - self-refresh (SR) mode
5076  *   - lines are large relative to FIFO size (buffer can hold up to 2)
5077  *   - lines are small relative to FIFO size (buffer can hold more than 2
5078  *     lines), so need to account for TLB latency
5079  *
5080  *   The normal calculation is:
5081  *     watermark = dotclock * bytes per pixel * latency
5082  *   where latency is platform & configuration dependent (we assume pessimal
5083  *   values here).
5084  *
5085  *   The SR calculation is:
5086  *     watermark = (trunc(latency/line time)+1) * surface width *
5087  *       bytes per pixel
5088  *   where
5089  *     line time = htotal / dotclock
5090  *     surface width = hdisplay for normal plane and 64 for cursor
5091  *   and latency is assumed to be high, as above.
5092  *
5093  * The final value programmed to the register should always be rounded up,
5094  * and include an extra 2 entries to account for clock crossings.
5095  *
5096  * We don't use the sprite, so we can ignore that.  And on Crestline we have
5097  * to set the non-SR watermarks to 8.
5098  */
5099 static void intel_update_watermarks(struct drm_device *dev)
5100 {
5101         struct drm_i915_private *dev_priv = dev->dev_private;
5102
5103         if (dev_priv->display.update_wm)
5104                 dev_priv->display.update_wm(dev);
5105 }
5106
5107 void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
5108                                     uint32_t sprite_width, int pixel_size)
5109 {
5110         struct drm_i915_private *dev_priv = dev->dev_private;
5111
5112         if (dev_priv->display.update_sprite_wm)
5113                 dev_priv->display.update_sprite_wm(dev, pipe, sprite_width,
5114                                                    pixel_size);
5115 }
5116
5117 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
5118 {
5119         if (i915_panel_use_ssc >= 0)
5120                 return i915_panel_use_ssc != 0;
5121         return dev_priv->lvds_use_ssc
5122                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
5123 }
5124
5125 /**
5126  * intel_choose_pipe_bpp_dither - figure out what color depth the pipe should send
5127  * @crtc: CRTC structure
5128  * @mode: requested mode
5129  *
5130  * A pipe may be connected to one or more outputs.  Based on the depth of the
5131  * attached framebuffer, choose a good color depth to use on the pipe.
5132  *
5133  * If possible, match the pipe depth to the fb depth.  In some cases, this
5134  * isn't ideal, because the connected output supports a lesser or restricted
5135  * set of depths.  Resolve that here:
5136  *    LVDS typically supports only 6bpc, so clamp down in that case
5137  *    HDMI supports only 8bpc or 12bpc, so clamp to 8bpc with dither for 10bpc
5138  *    Displays may support a restricted set as well, check EDID and clamp as
5139  *      appropriate.
5140  *    DP may want to dither down to 6bpc to fit larger modes
5141  *
5142  * RETURNS:
5143  * Dithering requirement (i.e. false if display bpc and pipe bpc match,
5144  * true if they don't match).
5145  */
5146 static bool intel_choose_pipe_bpp_dither(struct drm_crtc *crtc,
5147                                          unsigned int *pipe_bpp,
5148                                          struct drm_display_mode *mode)
5149 {
5150         struct drm_device *dev = crtc->dev;
5151         struct drm_i915_private *dev_priv = dev->dev_private;
5152         struct drm_encoder *encoder;
5153         struct drm_connector *connector;
5154         unsigned int display_bpc = UINT_MAX, bpc;
5155
5156         /* Walk the encoders & connectors on this crtc, get min bpc */
5157         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
5158                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
5159
5160                 if (encoder->crtc != crtc)
5161                         continue;
5162
5163                 if (intel_encoder->type == INTEL_OUTPUT_LVDS) {
5164                         unsigned int lvds_bpc;
5165
5166                         if ((I915_READ(PCH_LVDS) & LVDS_A3_POWER_MASK) ==
5167                             LVDS_A3_POWER_UP)
5168                                 lvds_bpc = 8;
5169                         else
5170                                 lvds_bpc = 6;
5171
5172                         if (lvds_bpc < display_bpc) {
5173                                 DRM_DEBUG_KMS("clamping display bpc (was %d) to LVDS (%d)\n", display_bpc, lvds_bpc);
5174                                 display_bpc = lvds_bpc;
5175                         }
5176                         continue;
5177                 }
5178
5179                 if (intel_encoder->type == INTEL_OUTPUT_EDP) {
5180                         /* Use VBT settings if we have an eDP panel */
5181                         unsigned int edp_bpc = dev_priv->edp.bpp / 3;
5182
5183                         if (edp_bpc < display_bpc) {
5184                                 DRM_DEBUG_KMS("clamping display bpc (was %d) to eDP (%d)\n", display_bpc, edp_bpc);
5185                                 display_bpc = edp_bpc;
5186                         }
5187                         continue;
5188                 }
5189
5190                 /* Not one of the known troublemakers, check the EDID */
5191                 list_for_each_entry(connector, &dev->mode_config.connector_list,
5192                                     head) {
5193                         if (connector->encoder != encoder)
5194                                 continue;
5195
5196                         /* Don't use an invalid EDID bpc value */
5197                         if (connector->display_info.bpc &&
5198                             connector->display_info.bpc < display_bpc) {
5199                                 DRM_DEBUG_KMS("clamping display bpc (was %d) to EDID reported max of %d\n", display_bpc, connector->display_info.bpc);
5200                                 display_bpc = connector->display_info.bpc;
5201                         }
5202                 }
5203
5204                 /*
5205                  * HDMI is either 12 or 8, so if the display lets 10bpc sneak
5206                  * through, clamp it down.  (Note: >12bpc will be caught below.)
5207                  */
5208                 if (intel_encoder->type == INTEL_OUTPUT_HDMI) {
5209                         if (display_bpc > 8 && display_bpc < 12) {
5210                                 DRM_DEBUG_KMS("forcing bpc to 12 for HDMI\n");
5211                                 display_bpc = 12;
5212                         } else {
5213                                 DRM_DEBUG_KMS("forcing bpc to 8 for HDMI\n");
5214                                 display_bpc = 8;
5215                         }
5216                 }
5217         }
5218
5219         if (mode->private_flags & INTEL_MODE_DP_FORCE_6BPC) {
5220                 DRM_DEBUG_KMS("Dithering DP to 6bpc\n");
5221                 display_bpc = 6;
5222         }
5223
5224         /*
5225          * We could just drive the pipe at the highest bpc all the time and
5226          * enable dithering as needed, but that costs bandwidth.  So choose
5227          * the minimum value that expresses the full color range of the fb but
5228          * also stays within the max display bpc discovered above.
5229          */
5230
5231         switch (crtc->fb->depth) {
5232         case 8:
5233                 bpc = 8; /* since we go through a colormap */
5234                 break;
5235         case 15:
5236         case 16:
5237                 bpc = 6; /* min is 18bpp */
5238                 break;
5239         case 24:
5240                 bpc = 8;
5241                 break;
5242         case 30:
5243                 bpc = 10;
5244                 break;
5245         case 48:
5246                 bpc = 12;
5247                 break;
5248         default:
5249                 DRM_DEBUG("unsupported depth, assuming 24 bits\n");
5250                 bpc = min((unsigned int)8, display_bpc);
5251                 break;
5252         }
5253
5254         display_bpc = min(display_bpc, bpc);
5255
5256         DRM_DEBUG_KMS("setting pipe bpc to %d (max display bpc %d)\n",
5257                       bpc, display_bpc);
5258
5259         *pipe_bpp = display_bpc * 3;
5260
5261         return display_bpc != bpc;
5262 }
5263
5264 static int i9xx_get_refclk(struct drm_crtc *crtc, int num_connectors)
5265 {
5266         struct drm_device *dev = crtc->dev;
5267         struct drm_i915_private *dev_priv = dev->dev_private;
5268         int refclk;
5269
5270         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
5271             intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
5272                 refclk = dev_priv->lvds_ssc_freq * 1000;
5273                 DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n",
5274                               refclk / 1000);
5275         } else if (!IS_GEN2(dev)) {
5276                 refclk = 96000;
5277         } else {
5278                 refclk = 48000;
5279         }
5280
5281         return refclk;
5282 }
5283
5284 static void i9xx_adjust_sdvo_tv_clock(struct drm_display_mode *adjusted_mode,
5285                                       intel_clock_t *clock)
5286 {
5287         /* SDVO TV has fixed PLL values depend on its clock range,
5288            this mirrors vbios setting. */
5289         if (adjusted_mode->clock >= 100000
5290             && adjusted_mode->clock < 140500) {
5291                 clock->p1 = 2;
5292                 clock->p2 = 10;
5293                 clock->n = 3;
5294                 clock->m1 = 16;
5295                 clock->m2 = 8;
5296         } else if (adjusted_mode->clock >= 140500
5297                    && adjusted_mode->clock <= 200000) {
5298                 clock->p1 = 1;
5299                 clock->p2 = 10;
5300                 clock->n = 6;
5301                 clock->m1 = 12;
5302                 clock->m2 = 8;
5303         }
5304 }
5305
5306 static void i9xx_update_pll_dividers(struct drm_crtc *crtc,
5307                                      intel_clock_t *clock,
5308                                      intel_clock_t *reduced_clock)
5309 {
5310         struct drm_device *dev = crtc->dev;
5311         struct drm_i915_private *dev_priv = dev->dev_private;
5312         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5313         int pipe = intel_crtc->pipe;
5314         u32 fp, fp2 = 0;
5315
5316         if (IS_PINEVIEW(dev)) {
5317                 fp = (1 << clock->n) << 16 | clock->m1 << 8 | clock->m2;
5318                 if (reduced_clock)
5319                         fp2 = (1 << reduced_clock->n) << 16 |
5320                                 reduced_clock->m1 << 8 | reduced_clock->m2;
5321         } else {
5322                 fp = clock->n << 16 | clock->m1 << 8 | clock->m2;
5323                 if (reduced_clock)
5324                         fp2 = reduced_clock->n << 16 | reduced_clock->m1 << 8 |
5325                                 reduced_clock->m2;
5326         }
5327
5328         I915_WRITE(FP0(pipe), fp);
5329
5330         intel_crtc->lowfreq_avail = false;
5331         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
5332             reduced_clock && i915_powersave) {
5333                 I915_WRITE(FP1(pipe), fp2);
5334                 intel_crtc->lowfreq_avail = true;
5335         } else {
5336                 I915_WRITE(FP1(pipe), fp);
5337         }
5338 }
5339
5340 static void intel_update_lvds(struct drm_crtc *crtc, intel_clock_t *clock,
5341                               struct drm_display_mode *adjusted_mode)
5342 {
5343         struct drm_device *dev = crtc->dev;
5344         struct drm_i915_private *dev_priv = dev->dev_private;
5345         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5346         int pipe = intel_crtc->pipe;
5347         u32 temp, lvds_sync = 0;
5348
5349         temp = I915_READ(LVDS);
5350         temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
5351         if (pipe == 1) {
5352                 temp |= LVDS_PIPEB_SELECT;
5353         } else {
5354                 temp &= ~LVDS_PIPEB_SELECT;
5355         }
5356         /* set the corresponsding LVDS_BORDER bit */
5357         temp |= dev_priv->lvds_border_bits;
5358         /* Set the B0-B3 data pairs corresponding to whether we're going to
5359          * set the DPLLs for dual-channel mode or not.
5360          */
5361         if (clock->p2 == 7)
5362                 temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP;
5363         else
5364                 temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP);
5365
5366         /* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP)
5367          * appropriately here, but we need to look more thoroughly into how
5368          * panels behave in the two modes.
5369          */
5370         /* set the dithering flag on LVDS as needed */
5371         if (INTEL_INFO(dev)->gen >= 4) {
5372                 if (dev_priv->lvds_dither)
5373                         temp |= LVDS_ENABLE_DITHER;
5374                 else
5375                         temp &= ~LVDS_ENABLE_DITHER;
5376         }
5377         if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
5378                 lvds_sync |= LVDS_HSYNC_POLARITY;
5379         if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
5380                 lvds_sync |= LVDS_VSYNC_POLARITY;
5381         if ((temp & (LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY))
5382             != lvds_sync) {
5383                 char flags[2] = "-+";
5384                 DRM_INFO("Changing LVDS panel from "
5385                          "(%chsync, %cvsync) to (%chsync, %cvsync)\n",
5386                          flags[!(temp & LVDS_HSYNC_POLARITY)],
5387                          flags[!(temp & LVDS_VSYNC_POLARITY)],
5388                          flags[!(lvds_sync & LVDS_HSYNC_POLARITY)],
5389                          flags[!(lvds_sync & LVDS_VSYNC_POLARITY)]);
5390                 temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
5391                 temp |= lvds_sync;
5392         }
5393         I915_WRITE(LVDS, temp);
5394 }
5395
5396 static void i9xx_update_pll(struct drm_crtc *crtc,
5397                             struct drm_display_mode *mode,
5398                             struct drm_display_mode *adjusted_mode,
5399                             intel_clock_t *clock, intel_clock_t *reduced_clock,
5400                             int num_connectors)
5401 {
5402         struct drm_device *dev = crtc->dev;
5403         struct drm_i915_private *dev_priv = dev->dev_private;
5404         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5405         int pipe = intel_crtc->pipe;
5406         u32 dpll;
5407         bool is_sdvo;
5408
5409         is_sdvo = intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO) ||
5410                 intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI);
5411
5412         dpll = DPLL_VGA_MODE_DIS;
5413
5414         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
5415                 dpll |= DPLLB_MODE_LVDS;
5416         else
5417                 dpll |= DPLLB_MODE_DAC_SERIAL;
5418         if (is_sdvo) {
5419                 int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
5420                 if (pixel_multiplier > 1) {
5421                         if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
5422                                 dpll |= (pixel_multiplier - 1) << SDVO_MULTIPLIER_SHIFT_HIRES;
5423                 }
5424                 dpll |= DPLL_DVO_HIGH_SPEED;
5425         }
5426         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT))
5427                 dpll |= DPLL_DVO_HIGH_SPEED;
5428
5429         /* compute bitmask from p1 value */
5430         if (IS_PINEVIEW(dev))
5431                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
5432         else {
5433                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
5434                 if (IS_G4X(dev) && reduced_clock)
5435                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
5436         }
5437         switch (clock->p2) {
5438         case 5:
5439                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
5440                 break;
5441         case 7:
5442                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
5443                 break;
5444         case 10:
5445                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
5446                 break;
5447         case 14:
5448                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
5449                 break;
5450         }
5451         if (INTEL_INFO(dev)->gen >= 4)
5452                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
5453
5454         if (is_sdvo && intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT))
5455                 dpll |= PLL_REF_INPUT_TVCLKINBC;
5456         else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT))
5457                 /* XXX: just matching BIOS for now */
5458                 /*      dpll |= PLL_REF_INPUT_TVCLKINBC; */
5459                 dpll |= 3;
5460         else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
5461                  intel_panel_use_ssc(dev_priv) && num_connectors < 2)
5462                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
5463         else
5464                 dpll |= PLL_REF_INPUT_DREFCLK;
5465
5466         dpll |= DPLL_VCO_ENABLE;
5467         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
5468         POSTING_READ(DPLL(pipe));
5469         udelay(150);
5470
5471         /* The LVDS pin pair needs to be on before the DPLLs are enabled.
5472          * This is an exception to the general rule that mode_set doesn't turn
5473          * things on.
5474          */
5475         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
5476                 intel_update_lvds(crtc, clock, adjusted_mode);
5477
5478         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT))
5479                 intel_dp_set_m_n(crtc, mode, adjusted_mode);
5480
5481         I915_WRITE(DPLL(pipe), dpll);
5482
5483         /* Wait for the clocks to stabilize. */
5484         POSTING_READ(DPLL(pipe));
5485         udelay(150);
5486
5487         if (INTEL_INFO(dev)->gen >= 4) {
5488                 u32 temp = 0;
5489                 if (is_sdvo) {
5490                         temp = intel_mode_get_pixel_multiplier(adjusted_mode);
5491                         if (temp > 1)
5492                                 temp = (temp - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
5493                         else
5494                                 temp = 0;
5495                 }
5496                 I915_WRITE(DPLL_MD(pipe), temp);
5497         } else {
5498                 /* The pixel multiplier can only be updated once the
5499                  * DPLL is enabled and the clocks are stable.
5500                  *
5501                  * So write it again.
5502                  */
5503                 I915_WRITE(DPLL(pipe), dpll);
5504         }
5505 }
5506
5507 static void i8xx_update_pll(struct drm_crtc *crtc,
5508                             struct drm_display_mode *adjusted_mode,
5509                             intel_clock_t *clock,
5510                             int num_connectors)
5511 {
5512         struct drm_device *dev = crtc->dev;
5513         struct drm_i915_private *dev_priv = dev->dev_private;
5514         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5515         int pipe = intel_crtc->pipe;
5516         u32 dpll;
5517
5518         dpll = DPLL_VGA_MODE_DIS;
5519
5520         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
5521                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
5522         } else {
5523                 if (clock->p1 == 2)
5524                         dpll |= PLL_P1_DIVIDE_BY_TWO;
5525                 else
5526                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
5527                 if (clock->p2 == 4)
5528                         dpll |= PLL_P2_DIVIDE_BY_4;
5529         }
5530
5531         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT))
5532                 /* XXX: just matching BIOS for now */
5533                 /*      dpll |= PLL_REF_INPUT_TVCLKINBC; */
5534                 dpll |= 3;
5535         else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
5536                  intel_panel_use_ssc(dev_priv) && num_connectors < 2)
5537                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
5538         else
5539                 dpll |= PLL_REF_INPUT_DREFCLK;
5540
5541         dpll |= DPLL_VCO_ENABLE;
5542         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
5543         POSTING_READ(DPLL(pipe));
5544         udelay(150);
5545
5546         I915_WRITE(DPLL(pipe), dpll);
5547
5548         /* Wait for the clocks to stabilize. */
5549         POSTING_READ(DPLL(pipe));
5550         udelay(150);
5551
5552         /* The LVDS pin pair needs to be on before the DPLLs are enabled.
5553          * This is an exception to the general rule that mode_set doesn't turn
5554          * things on.
5555          */
5556         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
5557                 intel_update_lvds(crtc, clock, adjusted_mode);
5558
5559         /* The pixel multiplier can only be updated once the
5560          * DPLL is enabled and the clocks are stable.
5561          *
5562          * So write it again.
5563          */
5564         I915_WRITE(DPLL(pipe), dpll);
5565 }
5566
5567 static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
5568                               struct drm_display_mode *mode,
5569                               struct drm_display_mode *adjusted_mode,
5570                               int x, int y,
5571                               struct drm_framebuffer *old_fb)
5572 {
5573         struct drm_device *dev = crtc->dev;
5574         struct drm_i915_private *dev_priv = dev->dev_private;
5575         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5576         int pipe = intel_crtc->pipe;
5577         int plane = intel_crtc->plane;
5578         int refclk, num_connectors = 0;
5579         intel_clock_t clock, reduced_clock;
5580         u32 dspcntr, pipeconf, vsyncshift;
5581         bool ok, has_reduced_clock = false, is_sdvo = false;
5582         bool is_lvds = false, is_tv = false, is_dp = false;
5583         struct drm_mode_config *mode_config = &dev->mode_config;
5584         struct intel_encoder *encoder;
5585         const intel_limit_t *limit;
5586         int ret;
5587
5588         list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
5589                 if (encoder->base.crtc != crtc)
5590                         continue;
5591
5592                 switch (encoder->type) {
5593                 case INTEL_OUTPUT_LVDS:
5594                         is_lvds = true;
5595                         break;
5596                 case INTEL_OUTPUT_SDVO:
5597                 case INTEL_OUTPUT_HDMI:
5598                         is_sdvo = true;
5599                         if (encoder->needs_tv_clock)
5600                                 is_tv = true;
5601                         break;
5602                 case INTEL_OUTPUT_TVOUT:
5603                         is_tv = true;
5604                         break;
5605                 case INTEL_OUTPUT_DISPLAYPORT:
5606                         is_dp = true;
5607                         break;
5608                 }
5609
5610                 num_connectors++;
5611         }
5612
5613         refclk = i9xx_get_refclk(crtc, num_connectors);
5614
5615         /*
5616          * Returns a set of divisors for the desired target clock with the given
5617          * refclk, or FALSE.  The returned values represent the clock equation:
5618          * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
5619          */
5620         limit = intel_limit(crtc, refclk);
5621         ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, NULL,
5622                              &clock);
5623         if (!ok) {
5624                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
5625                 return -EINVAL;
5626         }
5627
5628         /* Ensure that the cursor is valid for the new mode before changing... */
5629         intel_crtc_update_cursor(crtc, true);
5630
5631         if (is_lvds && dev_priv->lvds_downclock_avail) {
5632                 /*
5633                  * Ensure we match the reduced clock's P to the target clock.
5634                  * If the clocks don't match, we can't switch the display clock
5635                  * by using the FP0/FP1. In such case we will disable the LVDS
5636                  * downclock feature.
5637                 */
5638                 has_reduced_clock = limit->find_pll(limit, crtc,
5639                                                     dev_priv->lvds_downclock,
5640                                                     refclk,
5641                                                     &clock,
5642                                                     &reduced_clock);
5643         }
5644
5645         if (is_sdvo && is_tv)
5646                 i9xx_adjust_sdvo_tv_clock(adjusted_mode, &clock);
5647
5648         i9xx_update_pll_dividers(crtc, &clock, has_reduced_clock ?
5649                                  &reduced_clock : NULL);
5650
5651         if (IS_GEN2(dev))
5652                 i8xx_update_pll(crtc, adjusted_mode, &clock, num_connectors);
5653         else
5654                 i9xx_update_pll(crtc, mode, adjusted_mode, &clock,
5655                                 has_reduced_clock ? &reduced_clock : NULL,
5656                                 num_connectors);
5657
5658         /* setup pipeconf */
5659         pipeconf = I915_READ(PIPECONF(pipe));
5660
5661         /* Set up the display plane register */
5662         dspcntr = DISPPLANE_GAMMA_ENABLE;
5663
5664         if (pipe == 0)
5665                 dspcntr &= ~DISPPLANE_SEL_PIPE_MASK;
5666         else
5667                 dspcntr |= DISPPLANE_SEL_PIPE_B;
5668
5669         if (pipe == 0 && INTEL_INFO(dev)->gen < 4) {
5670                 /* Enable pixel doubling when the dot clock is > 90% of the (display)
5671                  * core speed.
5672                  *
5673                  * XXX: No double-wide on 915GM pipe B. Is that the only reason for the
5674                  * pipe == 0 check?
5675                  */
5676                 if (mode->clock >
5677                     dev_priv->display.get_display_clock_speed(dev) * 9 / 10)
5678                         pipeconf |= PIPECONF_DOUBLE_WIDE;
5679                 else
5680                         pipeconf &= ~PIPECONF_DOUBLE_WIDE;
5681         }
5682
5683         /* default to 8bpc */
5684         pipeconf &= ~(PIPECONF_BPP_MASK | PIPECONF_DITHER_EN);
5685         if (is_dp) {
5686                 if (mode->private_flags & INTEL_MODE_DP_FORCE_6BPC) {
5687                         pipeconf |= PIPECONF_BPP_6 |
5688                                     PIPECONF_DITHER_EN |
5689                                     PIPECONF_DITHER_TYPE_SP;
5690                 }
5691         }
5692
5693         DRM_DEBUG_KMS("Mode for pipe %c:\n", pipe == 0 ? 'A' : 'B');
5694         drm_mode_debug_printmodeline(mode);
5695
5696         if (HAS_PIPE_CXSR(dev)) {
5697                 if (intel_crtc->lowfreq_avail) {
5698                         DRM_DEBUG_KMS("enabling CxSR downclocking\n");
5699                         pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
5700                 } else {
5701                         DRM_DEBUG_KMS("disabling CxSR downclocking\n");
5702                         pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK;
5703                 }
5704         }
5705
5706         pipeconf &= ~PIPECONF_INTERLACE_MASK;
5707         if (!IS_GEN2(dev) &&
5708             adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
5709                 pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
5710                 /* the chip adds 2 halflines automatically */
5711                 adjusted_mode->crtc_vtotal -= 1;
5712                 adjusted_mode->crtc_vblank_end -= 1;
5713                 vsyncshift = adjusted_mode->crtc_hsync_start
5714                              - adjusted_mode->crtc_htotal/2;
5715         } else {
5716                 pipeconf |= PIPECONF_PROGRESSIVE;
5717                 vsyncshift = 0;
5718         }
5719
5720         if (!IS_GEN3(dev))
5721                 I915_WRITE(VSYNCSHIFT(pipe), vsyncshift);
5722
5723         I915_WRITE(HTOTAL(pipe),
5724                    (adjusted_mode->crtc_hdisplay - 1) |
5725                    ((adjusted_mode->crtc_htotal - 1) << 16));
5726         I915_WRITE(HBLANK(pipe),
5727                    (adjusted_mode->crtc_hblank_start - 1) |
5728                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
5729         I915_WRITE(HSYNC(pipe),
5730                    (adjusted_mode->crtc_hsync_start - 1) |
5731                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
5732
5733         I915_WRITE(VTOTAL(pipe),
5734                    (adjusted_mode->crtc_vdisplay - 1) |
5735                    ((adjusted_mode->crtc_vtotal - 1) << 16));
5736         I915_WRITE(VBLANK(pipe),
5737                    (adjusted_mode->crtc_vblank_start - 1) |
5738                    ((adjusted_mode->crtc_vblank_end - 1) << 16));
5739         I915_WRITE(VSYNC(pipe),
5740                    (adjusted_mode->crtc_vsync_start - 1) |
5741                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
5742
5743         /* pipesrc and dspsize control the size that is scaled from,
5744          * which should always be the user's requested size.
5745          */
5746         I915_WRITE(DSPSIZE(plane),
5747                    ((mode->vdisplay - 1) << 16) |
5748                    (mode->hdisplay - 1));
5749         I915_WRITE(DSPPOS(plane), 0);
5750         I915_WRITE(PIPESRC(pipe),
5751                    ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1));
5752
5753         I915_WRITE(PIPECONF(pipe), pipeconf);
5754         POSTING_READ(PIPECONF(pipe));
5755         intel_enable_pipe(dev_priv, pipe, false);
5756
5757         intel_wait_for_vblank(dev, pipe);
5758
5759         I915_WRITE(DSPCNTR(plane), dspcntr);
5760         POSTING_READ(DSPCNTR(plane));
5761         intel_enable_plane(dev_priv, plane, pipe);
5762
5763         ret = intel_pipe_set_base(crtc, x, y, old_fb);
5764
5765         intel_update_watermarks(dev);
5766
5767         return ret;
5768 }
5769
5770 /*
5771  * Initialize reference clocks when the driver loads
5772  */
5773 void ironlake_init_pch_refclk(struct drm_device *dev)
5774 {
5775         struct drm_i915_private *dev_priv = dev->dev_private;
5776         struct drm_mode_config *mode_config = &dev->mode_config;
5777         struct intel_encoder *encoder;
5778         u32 temp;
5779         bool has_lvds = false;
5780         bool has_cpu_edp = false;
5781         bool has_pch_edp = false;
5782         bool has_panel = false;
5783         bool has_ck505 = false;
5784         bool can_ssc = false;
5785
5786         /* We need to take the global config into account */
5787         list_for_each_entry(encoder, &mode_config->encoder_list,
5788                             base.head) {
5789                 switch (encoder->type) {
5790                 case INTEL_OUTPUT_LVDS:
5791                         has_panel = true;
5792                         has_lvds = true;
5793                         break;
5794                 case INTEL_OUTPUT_EDP:
5795                         has_panel = true;
5796                         if (intel_encoder_is_pch_edp(&encoder->base))
5797                                 has_pch_edp = true;
5798                         else
5799                                 has_cpu_edp = true;
5800                         break;
5801                 }
5802         }
5803
5804         if (HAS_PCH_IBX(dev)) {
5805                 has_ck505 = dev_priv->display_clock_mode;
5806                 can_ssc = has_ck505;
5807         } else {
5808                 has_ck505 = false;
5809                 can_ssc = true;
5810         }
5811
5812         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_pch_edp %d has_cpu_edp %d has_ck505 %d\n",
5813                       has_panel, has_lvds, has_pch_edp, has_cpu_edp,
5814                       has_ck505);
5815
5816         /* Ironlake: try to setup display ref clock before DPLL
5817          * enabling. This is only under driver's control after
5818          * PCH B stepping, previous chipset stepping should be
5819          * ignoring this setting.
5820          */
5821         temp = I915_READ(PCH_DREF_CONTROL);
5822         /* Always enable nonspread source */
5823         temp &= ~DREF_NONSPREAD_SOURCE_MASK;
5824
5825         if (has_ck505)
5826                 temp |= DREF_NONSPREAD_CK505_ENABLE;
5827         else
5828                 temp |= DREF_NONSPREAD_SOURCE_ENABLE;
5829
5830         if (has_panel) {
5831                 temp &= ~DREF_SSC_SOURCE_MASK;
5832                 temp |= DREF_SSC_SOURCE_ENABLE;
5833
5834                 /* SSC must be turned on before enabling the CPU output  */
5835                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
5836                         DRM_DEBUG_KMS("Using SSC on panel\n");
5837                         temp |= DREF_SSC1_ENABLE;
5838                 }
5839
5840                 /* Get SSC going before enabling the outputs */
5841                 I915_WRITE(PCH_DREF_CONTROL, temp);
5842                 POSTING_READ(PCH_DREF_CONTROL);
5843                 udelay(200);
5844
5845                 temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
5846
5847                 /* Enable CPU source on CPU attached eDP */
5848                 if (has_cpu_edp) {
5849                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
5850                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
5851                                 temp |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
5852                         }
5853                         else
5854                                 temp |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
5855                 } else
5856                         temp |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
5857
5858                 I915_WRITE(PCH_DREF_CONTROL, temp);
5859                 POSTING_READ(PCH_DREF_CONTROL);
5860                 udelay(200);
5861         } else {
5862                 DRM_DEBUG_KMS("Disabling SSC entirely\n");
5863
5864                 temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
5865
5866                 /* Turn off CPU output */
5867                 temp |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
5868
5869                 I915_WRITE(PCH_DREF_CONTROL, temp);
5870                 POSTING_READ(PCH_DREF_CONTROL);
5871                 udelay(200);
5872
5873                 /* Turn off the SSC source */
5874                 temp &= ~DREF_SSC_SOURCE_MASK;
5875                 temp |= DREF_SSC_SOURCE_DISABLE;
5876
5877                 /* Turn off SSC1 */
5878                 temp &= ~ DREF_SSC1_ENABLE;
5879
5880                 I915_WRITE(PCH_DREF_CONTROL, temp);
5881                 POSTING_READ(PCH_DREF_CONTROL);
5882                 udelay(200);
5883         }
5884 }
5885
5886 static int ironlake_get_refclk(struct drm_crtc *crtc)
5887 {
5888         struct drm_device *dev = crtc->dev;
5889         struct drm_i915_private *dev_priv = dev->dev_private;
5890         struct intel_encoder *encoder;
5891         struct drm_mode_config *mode_config = &dev->mode_config;
5892         struct intel_encoder *edp_encoder = NULL;
5893         int num_connectors = 0;
5894         bool is_lvds = false;
5895
5896         list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
5897                 if (encoder->base.crtc != crtc)
5898                         continue;
5899
5900                 switch (encoder->type) {
5901                 case INTEL_OUTPUT_LVDS:
5902                         is_lvds = true;
5903                         break;
5904                 case INTEL_OUTPUT_EDP:
5905                         edp_encoder = encoder;
5906                         break;
5907                 }
5908                 num_connectors++;
5909         }
5910
5911         if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
5912                 DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n",
5913                               dev_priv->lvds_ssc_freq);
5914                 return dev_priv->lvds_ssc_freq * 1000;
5915         }
5916
5917         return 120000;
5918 }
5919
5920 static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
5921                                   struct drm_display_mode *mode,
5922                                   struct drm_display_mode *adjusted_mode,
5923                                   int x, int y,
5924                                   struct drm_framebuffer *old_fb)
5925 {
5926         struct drm_device *dev = crtc->dev;
5927         struct drm_i915_private *dev_priv = dev->dev_private;
5928         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5929         int pipe = intel_crtc->pipe;
5930         int plane = intel_crtc->plane;
5931         int refclk, num_connectors = 0;
5932         intel_clock_t clock, reduced_clock;
5933         u32 dpll, fp = 0, fp2 = 0, dspcntr, pipeconf;
5934         bool ok, has_reduced_clock = false, is_sdvo = false;
5935         bool is_crt = false, is_lvds = false, is_tv = false, is_dp = false;
5936         struct intel_encoder *has_edp_encoder = NULL;
5937         struct drm_mode_config *mode_config = &dev->mode_config;
5938         struct intel_encoder *encoder;
5939         const intel_limit_t *limit;
5940         int ret;
5941         struct fdi_m_n m_n = {0};
5942         u32 temp;
5943         u32 lvds_sync = 0;
5944         int target_clock, pixel_multiplier, lane, link_bw, factor;
5945         unsigned int pipe_bpp;
5946         bool dither;
5947
5948         list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
5949                 if (encoder->base.crtc != crtc)
5950                         continue;
5951
5952                 switch (encoder->type) {
5953                 case INTEL_OUTPUT_LVDS:
5954                         is_lvds = true;
5955                         break;
5956                 case INTEL_OUTPUT_SDVO:
5957                 case INTEL_OUTPUT_HDMI:
5958                         is_sdvo = true;
5959                         if (encoder->needs_tv_clock)
5960                                 is_tv = true;
5961                         break;
5962                 case INTEL_OUTPUT_TVOUT:
5963                         is_tv = true;
5964                         break;
5965                 case INTEL_OUTPUT_ANALOG:
5966                         is_crt = true;
5967                         break;
5968                 case INTEL_OUTPUT_DISPLAYPORT:
5969                         is_dp = true;
5970                         break;
5971                 case INTEL_OUTPUT_EDP:
5972                         has_edp_encoder = encoder;
5973                         break;
5974                 }
5975
5976                 num_connectors++;
5977         }
5978
5979         refclk = ironlake_get_refclk(crtc);
5980
5981         /*
5982          * Returns a set of divisors for the desired target clock with the given
5983          * refclk, or FALSE.  The returned values represent the clock equation:
5984          * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
5985          */
5986         limit = intel_limit(crtc, refclk);
5987         ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, NULL,
5988                              &clock);
5989         if (!ok) {
5990                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
5991                 return -EINVAL;
5992         }
5993
5994         /* Ensure that the cursor is valid for the new mode before changing... */
5995         intel_crtc_update_cursor(crtc, true);
5996
5997         if (is_lvds && dev_priv->lvds_downclock_avail) {
5998                 /*
5999                  * Ensure we match the reduced clock's P to the target clock.
6000                  * If the clocks don't match, we can't switch the display clock
6001                  * by using the FP0/FP1. In such case we will disable the LVDS
6002                  * downclock feature.
6003                 */
6004                 has_reduced_clock = limit->find_pll(limit, crtc,
6005                                                     dev_priv->lvds_downclock,
6006                                                     refclk,
6007                                                     &clock,
6008                                                     &reduced_clock);
6009         }
6010         /* SDVO TV has fixed PLL values depend on its clock range,
6011            this mirrors vbios setting. */
6012         if (is_sdvo && is_tv) {
6013                 if (adjusted_mode->clock >= 100000
6014                     && adjusted_mode->clock < 140500) {
6015                         clock.p1 = 2;
6016                         clock.p2 = 10;
6017                         clock.n = 3;
6018                         clock.m1 = 16;
6019                         clock.m2 = 8;
6020                 } else if (adjusted_mode->clock >= 140500
6021                            && adjusted_mode->clock <= 200000) {
6022                         clock.p1 = 1;
6023                         clock.p2 = 10;
6024                         clock.n = 6;
6025                         clock.m1 = 12;
6026                         clock.m2 = 8;
6027                 }
6028         }
6029
6030         /* FDI link */
6031         pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
6032         lane = 0;
6033         /* CPU eDP doesn't require FDI link, so just set DP M/N
6034            according to current link config */
6035         if (has_edp_encoder &&
6036             !intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
6037                 target_clock = mode->clock;
6038                 intel_edp_link_config(has_edp_encoder,
6039                                       &lane, &link_bw);
6040         } else {
6041                 /* [e]DP over FDI requires target mode clock
6042                    instead of link clock */
6043                 if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base))
6044                         target_clock = mode->clock;
6045                 else
6046                         target_clock = adjusted_mode->clock;
6047
6048                 /* FDI is a binary signal running at ~2.7GHz, encoding
6049                  * each output octet as 10 bits. The actual frequency
6050                  * is stored as a divider into a 100MHz clock, and the
6051                  * mode pixel clock is stored in units of 1KHz.
6052                  * Hence the bw of each lane in terms of the mode signal
6053                  * is:
6054                  */
6055                 link_bw = intel_fdi_link_freq(dev) * MHz(100)/KHz(1)/10;
6056         }
6057
6058         /* determine panel color depth */
6059         temp = I915_READ(PIPECONF(pipe));
6060         temp &= ~PIPE_BPC_MASK;
6061         dither = intel_choose_pipe_bpp_dither(crtc, &pipe_bpp, mode);
6062         switch (pipe_bpp) {
6063         case 18:
6064                 temp |= PIPE_6BPC;
6065                 break;
6066         case 24:
6067                 temp |= PIPE_8BPC;
6068                 break;
6069         case 30:
6070                 temp |= PIPE_10BPC;
6071                 break;
6072         case 36:
6073                 temp |= PIPE_12BPC;
6074                 break;
6075         default:
6076                 WARN(1, "intel_choose_pipe_bpp returned invalid value %d\n",
6077                         pipe_bpp);
6078                 temp |= PIPE_8BPC;
6079                 pipe_bpp = 24;
6080                 break;
6081         }
6082
6083         intel_crtc->bpp = pipe_bpp;
6084         I915_WRITE(PIPECONF(pipe), temp);
6085
6086         if (!lane) {
6087                 /*
6088                  * Account for spread spectrum to avoid
6089                  * oversubscribing the link. Max center spread
6090                  * is 2.5%; use 5% for safety's sake.
6091                  */
6092                 u32 bps = target_clock * intel_crtc->bpp * 21 / 20;
6093                 lane = bps / (link_bw * 8) + 1;
6094         }
6095
6096         intel_crtc->fdi_lanes = lane;
6097
6098         if (pixel_multiplier > 1)
6099                 link_bw *= pixel_multiplier;
6100         ironlake_compute_m_n(intel_crtc->bpp, lane, target_clock, link_bw,
6101                              &m_n);
6102
6103         fp = clock.n << 16 | clock.m1 << 8 | clock.m2;
6104         if (has_reduced_clock)
6105                 fp2 = reduced_clock.n << 16 | reduced_clock.m1 << 8 |
6106                         reduced_clock.m2;
6107
6108         /* Enable autotuning of the PLL clock (if permissible) */
6109         factor = 21;
6110         if (is_lvds) {
6111                 if ((intel_panel_use_ssc(dev_priv) &&
6112                      dev_priv->lvds_ssc_freq == 100) ||
6113                     (I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP)
6114                         factor = 25;
6115         } else if (is_sdvo && is_tv)
6116                 factor = 20;
6117
6118         if (clock.m < factor * clock.n)
6119                 fp |= FP_CB_TUNE;
6120
6121         dpll = 0;
6122
6123         if (is_lvds)
6124                 dpll |= DPLLB_MODE_LVDS;
6125         else
6126                 dpll |= DPLLB_MODE_DAC_SERIAL;
6127         if (is_sdvo) {
6128                 int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
6129                 if (pixel_multiplier > 1) {
6130                         dpll |= (pixel_multiplier - 1) << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
6131                 }
6132                 dpll |= DPLL_DVO_HIGH_SPEED;
6133         }
6134         if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base))
6135                 dpll |= DPLL_DVO_HIGH_SPEED;
6136
6137         /* compute bitmask from p1 value */
6138         dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
6139         /* also FPA1 */
6140         dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
6141
6142         switch (clock.p2) {
6143         case 5:
6144                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
6145                 break;
6146         case 7:
6147                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
6148                 break;
6149         case 10:
6150                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
6151                 break;
6152         case 14:
6153                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
6154                 break;
6155         }
6156
6157         if (is_sdvo && is_tv)
6158                 dpll |= PLL_REF_INPUT_TVCLKINBC;
6159         else if (is_tv)
6160                 /* XXX: just matching BIOS for now */
6161                 /*      dpll |= PLL_REF_INPUT_TVCLKINBC; */
6162                 dpll |= 3;
6163         else if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
6164                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
6165         else
6166                 dpll |= PLL_REF_INPUT_DREFCLK;
6167
6168         /* setup pipeconf */
6169         pipeconf = I915_READ(PIPECONF(pipe));
6170
6171         /* Set up the display plane register */
6172         dspcntr = DISPPLANE_GAMMA_ENABLE;
6173
6174         DRM_DEBUG_KMS("Mode for pipe %d:\n", pipe);
6175         drm_mode_debug_printmodeline(mode);
6176
6177         /* PCH eDP needs FDI, but CPU eDP does not */
6178         if (!intel_crtc->no_pll) {
6179                 if (!has_edp_encoder ||
6180                     intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
6181                         I915_WRITE(PCH_FP0(pipe), fp);
6182                         I915_WRITE(PCH_DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
6183
6184                         POSTING_READ(PCH_DPLL(pipe));
6185                         udelay(150);
6186                 }
6187         } else {
6188                 if (dpll == (I915_READ(PCH_DPLL(0)) & 0x7fffffff) &&
6189                     fp == I915_READ(PCH_FP0(0))) {
6190                         intel_crtc->use_pll_a = true;
6191                         DRM_DEBUG_KMS("using pipe a dpll\n");
6192                 } else if (dpll == (I915_READ(PCH_DPLL(1)) & 0x7fffffff) &&
6193                            fp == I915_READ(PCH_FP0(1))) {
6194                         intel_crtc->use_pll_a = false;
6195                         DRM_DEBUG_KMS("using pipe b dpll\n");
6196                 } else {
6197                         DRM_DEBUG_KMS("no matching PLL configuration for pipe 2\n");
6198                         return -EINVAL;
6199                 }
6200         }
6201
6202         /* The LVDS pin pair needs to be on before the DPLLs are enabled.
6203          * This is an exception to the general rule that mode_set doesn't turn
6204          * things on.
6205          */
6206         if (is_lvds) {
6207                 temp = I915_READ(PCH_LVDS);
6208                 temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
6209                 if (HAS_PCH_CPT(dev)) {
6210                         temp &= ~PORT_TRANS_SEL_MASK;
6211                         temp |= PORT_TRANS_SEL_CPT(pipe);
6212                 } else {
6213                         if (pipe == 1)
6214                                 temp |= LVDS_PIPEB_SELECT;
6215                         else
6216                                 temp &= ~LVDS_PIPEB_SELECT;
6217                 }
6218
6219                 /* set the corresponsding LVDS_BORDER bit */
6220                 temp |= dev_priv->lvds_border_bits;
6221                 /* Set the B0-B3 data pairs corresponding to whether we're going to
6222                  * set the DPLLs for dual-channel mode or not.
6223                  */
6224                 if (clock.p2 == 7)
6225                         temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP;
6226                 else
6227                         temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP);
6228
6229                 /* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP)
6230                  * appropriately here, but we need to look more thoroughly into how
6231                  * panels behave in the two modes.
6232                  */
6233                 if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
6234                         lvds_sync |= LVDS_HSYNC_POLARITY;
6235                 if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
6236                         lvds_sync |= LVDS_VSYNC_POLARITY;
6237                 if ((temp & (LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY))
6238                     != lvds_sync) {
6239                         char flags[2] = "-+";
6240                         DRM_INFO("Changing LVDS panel from "
6241                                  "(%chsync, %cvsync) to (%chsync, %cvsync)\n",
6242                                  flags[!(temp & LVDS_HSYNC_POLARITY)],
6243                                  flags[!(temp & LVDS_VSYNC_POLARITY)],
6244                                  flags[!(lvds_sync & LVDS_HSYNC_POLARITY)],
6245                                  flags[!(lvds_sync & LVDS_VSYNC_POLARITY)]);
6246                         temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
6247                         temp |= lvds_sync;
6248                 }
6249                 I915_WRITE(PCH_LVDS, temp);
6250         }
6251
6252         pipeconf &= ~PIPECONF_DITHER_EN;
6253         pipeconf &= ~PIPECONF_DITHER_TYPE_MASK;
6254         if ((is_lvds && dev_priv->lvds_dither) || dither) {
6255                 pipeconf |= PIPECONF_DITHER_EN;
6256                 pipeconf |= PIPECONF_DITHER_TYPE_SP;
6257         }
6258         if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
6259                 intel_dp_set_m_n(crtc, mode, adjusted_mode);
6260         } else {
6261                 /* For non-DP output, clear any trans DP clock recovery setting.*/
6262                 I915_WRITE(TRANSDATA_M1(pipe), 0);
6263                 I915_WRITE(TRANSDATA_N1(pipe), 0);
6264                 I915_WRITE(TRANSDPLINK_M1(pipe), 0);
6265                 I915_WRITE(TRANSDPLINK_N1(pipe), 0);
6266         }
6267
6268         if (!intel_crtc->no_pll &&
6269             (!has_edp_encoder ||
6270              intel_encoder_is_pch_edp(&has_edp_encoder->base))) {
6271                 I915_WRITE(PCH_DPLL(pipe), dpll);
6272
6273                 /* Wait for the clocks to stabilize. */
6274                 POSTING_READ(PCH_DPLL(pipe));
6275                 udelay(150);
6276
6277                 /* The pixel multiplier can only be updated once the
6278                  * DPLL is enabled and the clocks are stable.
6279                  *
6280                  * So write it again.
6281                  */
6282                 I915_WRITE(PCH_DPLL(pipe), dpll);
6283         }
6284
6285         intel_crtc->lowfreq_avail = false;
6286         if (!intel_crtc->no_pll) {
6287                 if (is_lvds && has_reduced_clock && i915_powersave) {
6288                         I915_WRITE(PCH_FP1(pipe), fp2);
6289                         intel_crtc->lowfreq_avail = true;
6290                         if (HAS_PIPE_CXSR(dev)) {
6291                                 DRM_DEBUG_KMS("enabling CxSR downclocking\n");
6292                                 pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
6293                         }
6294                 } else {
6295                         I915_WRITE(PCH_FP1(pipe), fp);
6296                         if (HAS_PIPE_CXSR(dev)) {
6297                                 DRM_DEBUG_KMS("disabling CxSR downclocking\n");
6298                                 pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK;
6299                         }
6300                 }
6301         }
6302
6303         pipeconf &= ~PIPECONF_INTERLACE_MASK;
6304         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
6305                 pipeconf |= PIPECONF_INTERLACED_ILK;
6306                 /* the chip adds 2 halflines automatically */
6307                 adjusted_mode->crtc_vtotal -= 1;
6308                 adjusted_mode->crtc_vblank_end -= 1;
6309                 I915_WRITE(VSYNCSHIFT(pipe),
6310                            adjusted_mode->crtc_hsync_start
6311                            - adjusted_mode->crtc_htotal/2);
6312         } else {
6313                 pipeconf |= PIPECONF_PROGRESSIVE;
6314                 I915_WRITE(VSYNCSHIFT(pipe), 0);
6315         }
6316
6317         I915_WRITE(HTOTAL(pipe),
6318                    (adjusted_mode->crtc_hdisplay - 1) |
6319                    ((adjusted_mode->crtc_htotal - 1) << 16));
6320         I915_WRITE(HBLANK(pipe),
6321                    (adjusted_mode->crtc_hblank_start - 1) |
6322                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
6323         I915_WRITE(HSYNC(pipe),
6324                    (adjusted_mode->crtc_hsync_start - 1) |
6325                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
6326
6327         I915_WRITE(VTOTAL(pipe),
6328                    (adjusted_mode->crtc_vdisplay - 1) |
6329                    ((adjusted_mode->crtc_vtotal - 1) << 16));
6330         I915_WRITE(VBLANK(pipe),
6331                    (adjusted_mode->crtc_vblank_start - 1) |
6332                    ((adjusted_mode->crtc_vblank_end - 1) << 16));
6333         I915_WRITE(VSYNC(pipe),
6334                    (adjusted_mode->crtc_vsync_start - 1) |
6335                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
6336
6337         /* pipesrc controls the size that is scaled from, which should
6338          * always be the user's requested size.
6339          */
6340         I915_WRITE(PIPESRC(pipe),
6341                    ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1));
6342
6343         I915_WRITE(PIPE_DATA_M1(pipe), TU_SIZE(m_n.tu) | m_n.gmch_m);
6344         I915_WRITE(PIPE_DATA_N1(pipe), m_n.gmch_n);
6345         I915_WRITE(PIPE_LINK_M1(pipe), m_n.link_m);
6346         I915_WRITE(PIPE_LINK_N1(pipe), m_n.link_n);
6347
6348         if (has_edp_encoder &&
6349             !intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
6350                 ironlake_set_pll_edp(crtc, adjusted_mode->clock);
6351         }
6352
6353         I915_WRITE(PIPECONF(pipe), pipeconf);
6354         POSTING_READ(PIPECONF(pipe));
6355
6356         intel_wait_for_vblank(dev, pipe);
6357
6358         I915_WRITE(DSPCNTR(plane), dspcntr);
6359         POSTING_READ(DSPCNTR(plane));
6360
6361         ret = intel_pipe_set_base(crtc, x, y, old_fb);
6362
6363         intel_update_watermarks(dev);
6364
6365         return ret;
6366 }
6367
6368 static int intel_crtc_mode_set(struct drm_crtc *crtc,
6369                                struct drm_display_mode *mode,
6370                                struct drm_display_mode *adjusted_mode,
6371                                int x, int y,
6372                                struct drm_framebuffer *old_fb)
6373 {
6374         struct drm_device *dev = crtc->dev;
6375         struct drm_i915_private *dev_priv = dev->dev_private;
6376         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6377         int pipe = intel_crtc->pipe;
6378         int ret;
6379
6380         drm_vblank_pre_modeset(dev, pipe);
6381
6382         ret = dev_priv->display.crtc_mode_set(crtc, mode, adjusted_mode,
6383                                               x, y, old_fb);
6384         drm_vblank_post_modeset(dev, pipe);
6385
6386         if (ret)
6387                 intel_crtc->dpms_mode = DRM_MODE_DPMS_OFF;
6388         else
6389                 intel_crtc->dpms_mode = DRM_MODE_DPMS_ON;
6390
6391         return ret;
6392 }
6393
6394 static bool intel_eld_uptodate(struct drm_connector *connector,
6395                                int reg_eldv, uint32_t bits_eldv,
6396                                int reg_elda, uint32_t bits_elda,
6397                                int reg_edid)
6398 {
6399         struct drm_i915_private *dev_priv = connector->dev->dev_private;
6400         uint8_t *eld = connector->eld;
6401         uint32_t i;
6402
6403         i = I915_READ(reg_eldv);
6404         i &= bits_eldv;
6405
6406         if (!eld[0])
6407                 return !i;
6408
6409         if (!i)
6410                 return false;
6411
6412         i = I915_READ(reg_elda);
6413         i &= ~bits_elda;
6414         I915_WRITE(reg_elda, i);
6415
6416         for (i = 0; i < eld[2]; i++)
6417                 if (I915_READ(reg_edid) != *((uint32_t *)eld + i))
6418                         return false;
6419
6420         return true;
6421 }
6422
6423 static void g4x_write_eld(struct drm_connector *connector,
6424                           struct drm_crtc *crtc)
6425 {
6426         struct drm_i915_private *dev_priv = connector->dev->dev_private;
6427         uint8_t *eld = connector->eld;
6428         uint32_t eldv;
6429         uint32_t len;
6430         uint32_t i;
6431
6432         i = I915_READ(G4X_AUD_VID_DID);
6433
6434         if (i == INTEL_AUDIO_DEVBLC || i == INTEL_AUDIO_DEVCL)
6435                 eldv = G4X_ELDV_DEVCL_DEVBLC;
6436         else
6437                 eldv = G4X_ELDV_DEVCTG;
6438
6439         if (intel_eld_uptodate(connector,
6440                                G4X_AUD_CNTL_ST, eldv,
6441                                G4X_AUD_CNTL_ST, G4X_ELD_ADDR,
6442                                G4X_HDMIW_HDMIEDID))
6443                 return;
6444
6445         i = I915_READ(G4X_AUD_CNTL_ST);
6446         i &= ~(eldv | G4X_ELD_ADDR);
6447         len = (i >> 9) & 0x1f;          /* ELD buffer size */
6448         I915_WRITE(G4X_AUD_CNTL_ST, i);
6449
6450         if (!eld[0])
6451                 return;
6452
6453         len = min_t(uint8_t, eld[2], len);
6454         DRM_DEBUG_DRIVER("ELD size %d\n", len);
6455         for (i = 0; i < len; i++)
6456                 I915_WRITE(G4X_HDMIW_HDMIEDID, *((uint32_t *)eld + i));
6457
6458         i = I915_READ(G4X_AUD_CNTL_ST);
6459         i |= eldv;
6460         I915_WRITE(G4X_AUD_CNTL_ST, i);
6461 }
6462
6463 static void ironlake_write_eld(struct drm_connector *connector,
6464                                      struct drm_crtc *crtc)
6465 {
6466         struct drm_i915_private *dev_priv = connector->dev->dev_private;
6467         uint8_t *eld = connector->eld;
6468         uint32_t eldv;
6469         uint32_t i;
6470         int len;
6471         int hdmiw_hdmiedid;
6472         int aud_config;
6473         int aud_cntl_st;
6474         int aud_cntrl_st2;
6475
6476         if (HAS_PCH_IBX(connector->dev)) {
6477                 hdmiw_hdmiedid = IBX_HDMIW_HDMIEDID_A;
6478                 aud_config = IBX_AUD_CONFIG_A;
6479                 aud_cntl_st = IBX_AUD_CNTL_ST_A;
6480                 aud_cntrl_st2 = IBX_AUD_CNTL_ST2;
6481         } else {
6482                 hdmiw_hdmiedid = CPT_HDMIW_HDMIEDID_A;
6483                 aud_config = CPT_AUD_CONFIG_A;
6484                 aud_cntl_st = CPT_AUD_CNTL_ST_A;
6485                 aud_cntrl_st2 = CPT_AUD_CNTRL_ST2;
6486         }
6487
6488         i = to_intel_crtc(crtc)->pipe;
6489         hdmiw_hdmiedid += i * 0x100;
6490         aud_cntl_st += i * 0x100;
6491         aud_config += i * 0x100;
6492
6493         DRM_DEBUG_DRIVER("ELD on pipe %c\n", pipe_name(i));
6494
6495         i = I915_READ(aud_cntl_st);
6496         i = (i >> 29) & 0x3;            /* DIP_Port_Select, 0x1 = PortB */
6497         if (!i) {
6498                 DRM_DEBUG_DRIVER("Audio directed to unknown port\n");
6499                 /* operate blindly on all ports */
6500                 eldv = IBX_ELD_VALIDB;
6501                 eldv |= IBX_ELD_VALIDB << 4;
6502                 eldv |= IBX_ELD_VALIDB << 8;
6503         } else {
6504                 DRM_DEBUG_DRIVER("ELD on port %c\n", 'A' + i);
6505                 eldv = IBX_ELD_VALIDB << ((i - 1) * 4);
6506         }
6507
6508         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
6509                 DRM_DEBUG_DRIVER("ELD: DisplayPort detected\n");
6510                 eld[5] |= (1 << 2);     /* Conn_Type, 0x1 = DisplayPort */
6511                 I915_WRITE(aud_config, AUD_CONFIG_N_VALUE_INDEX); /* 0x1 = DP */
6512         } else
6513                 I915_WRITE(aud_config, 0);
6514
6515         if (intel_eld_uptodate(connector,
6516                                aud_cntrl_st2, eldv,
6517                                aud_cntl_st, IBX_ELD_ADDRESS,
6518                                hdmiw_hdmiedid))
6519                 return;
6520
6521         i = I915_READ(aud_cntrl_st2);
6522         i &= ~eldv;
6523         I915_WRITE(aud_cntrl_st2, i);
6524
6525         if (!eld[0])
6526                 return;
6527
6528         i = I915_READ(aud_cntl_st);
6529         i &= ~IBX_ELD_ADDRESS;
6530         I915_WRITE(aud_cntl_st, i);
6531
6532         len = min_t(uint8_t, eld[2], 21);       /* 84 bytes of hw ELD buffer */
6533         DRM_DEBUG_DRIVER("ELD size %d\n", len);
6534         for (i = 0; i < len; i++)
6535                 I915_WRITE(hdmiw_hdmiedid, *((uint32_t *)eld + i));
6536
6537         i = I915_READ(aud_cntrl_st2);
6538         i |= eldv;
6539         I915_WRITE(aud_cntrl_st2, i);
6540 }
6541
6542 void intel_write_eld(struct drm_encoder *encoder,
6543                      struct drm_display_mode *mode)
6544 {
6545         struct drm_crtc *crtc = encoder->crtc;
6546         struct drm_connector *connector;
6547         struct drm_device *dev = encoder->dev;
6548         struct drm_i915_private *dev_priv = dev->dev_private;
6549
6550         connector = drm_select_eld(encoder, mode);
6551         if (!connector)
6552                 return;
6553
6554         DRM_DEBUG_DRIVER("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
6555                          connector->base.id,
6556                          drm_get_connector_name(connector),
6557                          connector->encoder->base.id,
6558                          drm_get_encoder_name(connector->encoder));
6559
6560         connector->eld[6] = drm_av_sync_delay(connector, mode) / 2;
6561
6562         if (dev_priv->display.write_eld)
6563                 dev_priv->display.write_eld(connector, crtc);
6564 }
6565
6566 /** Loads the palette/gamma unit for the CRTC with the prepared values */
6567 void intel_crtc_load_lut(struct drm_crtc *crtc)
6568 {
6569         struct drm_device *dev = crtc->dev;
6570         struct drm_i915_private *dev_priv = dev->dev_private;
6571         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6572         int palreg = PALETTE(intel_crtc->pipe);
6573         int i;
6574
6575         /* The clocks have to be on to load the palette. */
6576         if (!crtc->enabled)
6577                 return;
6578
6579         /* use legacy palette for Ironlake */
6580         if (HAS_PCH_SPLIT(dev))
6581                 palreg = LGC_PALETTE(intel_crtc->pipe);
6582
6583         for (i = 0; i < 256; i++) {
6584                 I915_WRITE(palreg + 4 * i,
6585                            (intel_crtc->lut_r[i] << 16) |
6586                            (intel_crtc->lut_g[i] << 8) |
6587                            intel_crtc->lut_b[i]);
6588         }
6589 }
6590
6591 static void i845_update_cursor(struct drm_crtc *crtc, u32 base)
6592 {
6593         struct drm_device *dev = crtc->dev;
6594         struct drm_i915_private *dev_priv = dev->dev_private;
6595         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6596         bool visible = base != 0;
6597         u32 cntl;
6598
6599         if (intel_crtc->cursor_visible == visible)
6600                 return;
6601
6602         cntl = I915_READ(_CURACNTR);
6603         if (visible) {
6604                 /* On these chipsets we can only modify the base whilst
6605                  * the cursor is disabled.
6606                  */
6607                 I915_WRITE(_CURABASE, base);
6608
6609                 cntl &= ~(CURSOR_FORMAT_MASK);
6610                 /* XXX width must be 64, stride 256 => 0x00 << 28 */
6611                 cntl |= CURSOR_ENABLE |
6612                         CURSOR_GAMMA_ENABLE |
6613                         CURSOR_FORMAT_ARGB;
6614         } else
6615                 cntl &= ~(CURSOR_ENABLE | CURSOR_GAMMA_ENABLE);
6616         I915_WRITE(_CURACNTR, cntl);
6617
6618         intel_crtc->cursor_visible = visible;
6619 }
6620
6621 static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
6622 {
6623         struct drm_device *dev = crtc->dev;
6624         struct drm_i915_private *dev_priv = dev->dev_private;
6625         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6626         int pipe = intel_crtc->pipe;
6627         bool visible = base != 0;
6628
6629         if (intel_crtc->cursor_visible != visible) {
6630                 uint32_t cntl = I915_READ(CURCNTR(pipe));
6631                 if (base) {
6632                         cntl &= ~(CURSOR_MODE | MCURSOR_PIPE_SELECT);
6633                         cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE;
6634                         cntl |= pipe << 28; /* Connect to correct pipe */
6635                 } else {
6636                         cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE);
6637                         cntl |= CURSOR_MODE_DISABLE;
6638                 }
6639                 I915_WRITE(CURCNTR(pipe), cntl);
6640
6641                 intel_crtc->cursor_visible = visible;
6642         }
6643         /* and commit changes on next vblank */
6644         I915_WRITE(CURBASE(pipe), base);
6645 }
6646
6647 static void ivb_update_cursor(struct drm_crtc *crtc, u32 base)
6648 {
6649         struct drm_device *dev = crtc->dev;
6650         struct drm_i915_private *dev_priv = dev->dev_private;
6651         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6652         int pipe = intel_crtc->pipe;
6653         bool visible = base != 0;
6654
6655         if (intel_crtc->cursor_visible != visible) {
6656                 uint32_t cntl = I915_READ(CURCNTR_IVB(pipe));
6657                 if (base) {
6658                         cntl &= ~CURSOR_MODE;
6659                         cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE;
6660                 } else {
6661                         cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE);
6662                         cntl |= CURSOR_MODE_DISABLE;
6663                 }
6664                 I915_WRITE(CURCNTR_IVB(pipe), cntl);
6665
6666                 intel_crtc->cursor_visible = visible;
6667         }
6668         /* and commit changes on next vblank */
6669         I915_WRITE(CURBASE_IVB(pipe), base);
6670 }
6671
6672 /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
6673 static void intel_crtc_update_cursor(struct drm_crtc *crtc,
6674                                      bool on)
6675 {
6676         struct drm_device *dev = crtc->dev;
6677         struct drm_i915_private *dev_priv = dev->dev_private;
6678         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6679         int pipe = intel_crtc->pipe;
6680         int x = intel_crtc->cursor_x;
6681         int y = intel_crtc->cursor_y;
6682         u32 base, pos;
6683         bool visible;
6684
6685         pos = 0;
6686
6687         if (on && crtc->enabled && crtc->fb) {
6688                 base = intel_crtc->cursor_addr;
6689                 if (x > (int) crtc->fb->width)
6690                         base = 0;
6691
6692                 if (y > (int) crtc->fb->height)
6693                         base = 0;
6694         } else
6695                 base = 0;
6696
6697         if (x < 0) {
6698                 if (x + intel_crtc->cursor_width < 0)
6699                         base = 0;
6700
6701                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
6702                 x = -x;
6703         }
6704         pos |= x << CURSOR_X_SHIFT;
6705
6706         if (y < 0) {
6707                 if (y + intel_crtc->cursor_height < 0)
6708                         base = 0;
6709
6710                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
6711                 y = -y;
6712         }
6713         pos |= y << CURSOR_Y_SHIFT;
6714
6715         visible = base != 0;
6716         if (!visible && !intel_crtc->cursor_visible)
6717                 return;
6718
6719         if (IS_IVYBRIDGE(dev)) {
6720                 I915_WRITE(CURPOS_IVB(pipe), pos);
6721                 ivb_update_cursor(crtc, base);
6722         } else {
6723                 I915_WRITE(CURPOS(pipe), pos);
6724                 if (IS_845G(dev) || IS_I865G(dev))
6725                         i845_update_cursor(crtc, base);
6726                 else
6727                         i9xx_update_cursor(crtc, base);
6728         }
6729
6730         if (visible)
6731                 intel_mark_busy(dev, to_intel_framebuffer(crtc->fb)->obj);
6732 }
6733
6734 static int intel_crtc_cursor_set(struct drm_crtc *crtc,
6735                                  struct drm_file *file,
6736                                  uint32_t handle,
6737                                  uint32_t width, uint32_t height)
6738 {
6739         struct drm_device *dev = crtc->dev;
6740         struct drm_i915_private *dev_priv = dev->dev_private;
6741         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6742         struct drm_i915_gem_object *obj;
6743         uint32_t addr;
6744         int ret;
6745
6746         DRM_DEBUG_KMS("\n");
6747
6748         /* if we want to turn off the cursor ignore width and height */
6749         if (!handle) {
6750                 DRM_DEBUG_KMS("cursor off\n");
6751                 addr = 0;
6752                 obj = NULL;
6753                 mutex_lock(&dev->struct_mutex);
6754                 goto finish;
6755         }
6756
6757         /* Currently we only support 64x64 cursors */
6758         if (width != 64 || height != 64) {
6759                 DRM_ERROR("we currently only support 64x64 cursors\n");
6760                 return -EINVAL;
6761         }
6762
6763         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
6764         if (&obj->base == NULL)
6765                 return -ENOENT;
6766
6767         if (obj->base.size < width * height * 4) {
6768                 DRM_ERROR("buffer is to small\n");
6769                 ret = -ENOMEM;
6770                 goto fail;
6771         }
6772
6773         /* we only need to pin inside GTT if cursor is non-phy */
6774         mutex_lock(&dev->struct_mutex);
6775         if (!dev_priv->info->cursor_needs_physical) {
6776                 if (obj->tiling_mode) {
6777                         DRM_ERROR("cursor cannot be tiled\n");
6778                         ret = -EINVAL;
6779                         goto fail_locked;
6780                 }
6781
6782                 ret = i915_gem_object_pin_to_display_plane(obj, 0, NULL);
6783                 if (ret) {
6784                         DRM_ERROR("failed to move cursor bo into the GTT\n");
6785                         goto fail_locked;
6786                 }
6787
6788                 ret = i915_gem_object_put_fence(obj);
6789                 if (ret) {
6790                         DRM_ERROR("failed to release fence for cursor");
6791                         goto fail_unpin;
6792                 }
6793
6794                 addr = obj->gtt_offset;
6795         } else {
6796                 int align = IS_I830(dev) ? 16 * 1024 : 256;
6797                 ret = i915_gem_attach_phys_object(dev, obj,
6798                                                   (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1,
6799                                                   align);
6800                 if (ret) {
6801                         DRM_ERROR("failed to attach phys object\n");
6802                         goto fail_locked;
6803                 }
6804                 addr = obj->phys_obj->handle->busaddr;
6805         }
6806
6807         if (IS_GEN2(dev))
6808                 I915_WRITE(CURSIZE, (height << 12) | width);
6809
6810  finish:
6811         if (intel_crtc->cursor_bo) {
6812                 if (dev_priv->info->cursor_needs_physical) {
6813                         if (intel_crtc->cursor_bo != obj)
6814                                 i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo);
6815                 } else
6816                         i915_gem_object_unpin(intel_crtc->cursor_bo);
6817                 drm_gem_object_unreference(&intel_crtc->cursor_bo->base);
6818         }
6819
6820         mutex_unlock(&dev->struct_mutex);
6821
6822         intel_crtc->cursor_addr = addr;
6823         intel_crtc->cursor_bo = obj;
6824         intel_crtc->cursor_width = width;
6825         intel_crtc->cursor_height = height;
6826
6827         intel_crtc_update_cursor(crtc, true);
6828
6829         return 0;
6830 fail_unpin:
6831         i915_gem_object_unpin(obj);
6832 fail_locked:
6833         mutex_unlock(&dev->struct_mutex);
6834 fail:
6835         drm_gem_object_unreference_unlocked(&obj->base);
6836         return ret;
6837 }
6838
6839 static int intel_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
6840 {
6841         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6842
6843         intel_crtc->cursor_x = x;
6844         intel_crtc->cursor_y = y;
6845
6846         intel_crtc_update_cursor(crtc, true);
6847
6848         return 0;
6849 }
6850
6851 /** Sets the color ramps on behalf of RandR */
6852 void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
6853                                  u16 blue, int regno)
6854 {
6855         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6856
6857         intel_crtc->lut_r[regno] = red >> 8;
6858         intel_crtc->lut_g[regno] = green >> 8;
6859         intel_crtc->lut_b[regno] = blue >> 8;
6860 }
6861
6862 void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
6863                              u16 *blue, int regno)
6864 {
6865         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6866
6867         *red = intel_crtc->lut_r[regno] << 8;
6868         *green = intel_crtc->lut_g[regno] << 8;
6869         *blue = intel_crtc->lut_b[regno] << 8;
6870 }
6871
6872 static void intel_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
6873                                  u16 *blue, uint32_t start, uint32_t size)
6874 {
6875         int end = (start + size > 256) ? 256 : start + size, i;
6876         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6877
6878         for (i = start; i < end; i++) {
6879                 intel_crtc->lut_r[i] = red[i] >> 8;
6880                 intel_crtc->lut_g[i] = green[i] >> 8;
6881                 intel_crtc->lut_b[i] = blue[i] >> 8;
6882         }
6883
6884         intel_crtc_load_lut(crtc);
6885 }
6886
6887 /**
6888  * Get a pipe with a simple mode set on it for doing load-based monitor
6889  * detection.
6890  *
6891  * It will be up to the load-detect code to adjust the pipe as appropriate for
6892  * its requirements.  The pipe will be connected to no other encoders.
6893  *
6894  * Currently this code will only succeed if there is a pipe with no encoders
6895  * configured for it.  In the future, it could choose to temporarily disable
6896  * some outputs to free up a pipe for its use.
6897  *
6898  * \return crtc, or NULL if no pipes are available.
6899  */
6900
6901 /* VESA 640x480x72Hz mode to set on the pipe */
6902 static struct drm_display_mode load_detect_mode = {
6903         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
6904                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
6905 };
6906
6907 static struct drm_framebuffer *
6908 intel_framebuffer_create(struct drm_device *dev,
6909                          struct drm_mode_fb_cmd2 *mode_cmd,
6910                          struct drm_i915_gem_object *obj)
6911 {
6912         struct intel_framebuffer *intel_fb;
6913         int ret;
6914
6915         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
6916         if (!intel_fb) {
6917                 drm_gem_object_unreference_unlocked(&obj->base);
6918                 return ERR_PTR(-ENOMEM);
6919         }
6920
6921         ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
6922         if (ret) {
6923                 drm_gem_object_unreference_unlocked(&obj->base);
6924                 kfree(intel_fb);
6925                 return ERR_PTR(ret);
6926         }
6927
6928         return &intel_fb->base;
6929 }
6930
6931 static u32
6932 intel_framebuffer_pitch_for_width(int width, int bpp)
6933 {
6934         u32 pitch = DIV_ROUND_UP(width * bpp, 8);
6935         return ALIGN(pitch, 64);
6936 }
6937
6938 static u32
6939 intel_framebuffer_size_for_mode(struct drm_display_mode *mode, int bpp)
6940 {
6941         u32 pitch = intel_framebuffer_pitch_for_width(mode->hdisplay, bpp);
6942         return ALIGN(pitch * mode->vdisplay, PAGE_SIZE);
6943 }
6944
6945 static struct drm_framebuffer *
6946 intel_framebuffer_create_for_mode(struct drm_device *dev,
6947                                   struct drm_display_mode *mode,
6948                                   int depth, int bpp)
6949 {
6950         struct drm_i915_gem_object *obj;
6951         struct drm_mode_fb_cmd2 mode_cmd;
6952
6953         obj = i915_gem_alloc_object(dev,
6954                                     intel_framebuffer_size_for_mode(mode, bpp));
6955         if (obj == NULL)
6956                 return ERR_PTR(-ENOMEM);
6957
6958         mode_cmd.width = mode->hdisplay;
6959         mode_cmd.height = mode->vdisplay;
6960         mode_cmd.pitches[0] = intel_framebuffer_pitch_for_width(mode_cmd.width,
6961                                                                 bpp);
6962         mode_cmd.pixel_format = 0;
6963
6964         return intel_framebuffer_create(dev, &mode_cmd, obj);
6965 }
6966
6967 static struct drm_framebuffer *
6968 mode_fits_in_fbdev(struct drm_device *dev,
6969                    struct drm_display_mode *mode)
6970 {
6971         struct drm_i915_private *dev_priv = dev->dev_private;
6972         struct drm_i915_gem_object *obj;
6973         struct drm_framebuffer *fb;
6974
6975         if (dev_priv->fbdev == NULL)
6976                 return NULL;
6977
6978         obj = dev_priv->fbdev->ifb.obj;
6979         if (obj == NULL)
6980                 return NULL;
6981
6982         fb = &dev_priv->fbdev->ifb.base;
6983         if (fb->pitches[0] < intel_framebuffer_pitch_for_width(mode->hdisplay,
6984                                                                fb->bits_per_pixel))
6985                 return NULL;
6986
6987         if (obj->base.size < mode->vdisplay * fb->pitches[0])
6988                 return NULL;
6989
6990         return fb;
6991 }
6992
6993 bool intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
6994                                 struct drm_connector *connector,
6995                                 struct drm_display_mode *mode,
6996                                 struct intel_load_detect_pipe *old)
6997 {
6998         struct intel_crtc *intel_crtc;
6999         struct drm_crtc *possible_crtc;
7000         struct drm_encoder *encoder = &intel_encoder->base;
7001         struct drm_crtc *crtc = NULL;
7002         struct drm_device *dev = encoder->dev;
7003         struct drm_framebuffer *old_fb;
7004         int i = -1;
7005
7006         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
7007                       connector->base.id, drm_get_connector_name(connector),
7008                       encoder->base.id, drm_get_encoder_name(encoder));
7009
7010         /*
7011          * Algorithm gets a little messy:
7012          *
7013          *   - if the connector already has an assigned crtc, use it (but make
7014          *     sure it's on first)
7015          *
7016          *   - try to find the first unused crtc that can drive this connector,
7017          *     and use that if we find one
7018          */
7019
7020         /* See if we already have a CRTC for this connector */
7021         if (encoder->crtc) {
7022                 crtc = encoder->crtc;
7023
7024                 intel_crtc = to_intel_crtc(crtc);
7025                 old->dpms_mode = intel_crtc->dpms_mode;
7026                 old->load_detect_temp = false;
7027
7028                 /* Make sure the crtc and connector are running */
7029                 if (intel_crtc->dpms_mode != DRM_MODE_DPMS_ON) {
7030                         struct drm_encoder_helper_funcs *encoder_funcs;
7031                         struct drm_crtc_helper_funcs *crtc_funcs;
7032
7033                         crtc_funcs = crtc->helper_private;
7034                         crtc_funcs->dpms(crtc, DRM_MODE_DPMS_ON);
7035
7036                         encoder_funcs = encoder->helper_private;
7037                         encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON);
7038                 }
7039
7040                 return true;
7041         }
7042
7043         /* Find an unused one (if possible) */
7044         list_for_each_entry(possible_crtc, &dev->mode_config.crtc_list, head) {
7045                 i++;
7046                 if (!(encoder->possible_crtcs & (1 << i)))
7047                         continue;
7048                 if (!possible_crtc->enabled) {
7049                         crtc = possible_crtc;
7050                         break;
7051                 }
7052         }
7053
7054         /*
7055          * If we didn't find an unused CRTC, don't use any.
7056          */
7057         if (!crtc) {
7058                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
7059                 return false;
7060         }
7061
7062         encoder->crtc = crtc;
7063         connector->encoder = encoder;
7064
7065         intel_crtc = to_intel_crtc(crtc);
7066         old->dpms_mode = intel_crtc->dpms_mode;
7067         old->load_detect_temp = true;
7068         old->release_fb = NULL;
7069
7070         if (!mode)
7071                 mode = &load_detect_mode;
7072
7073         old_fb = crtc->fb;
7074
7075         /* We need a framebuffer large enough to accommodate all accesses
7076          * that the plane may generate whilst we perform load detection.
7077          * We can not rely on the fbcon either being present (we get called
7078          * during its initialisation to detect all boot displays, or it may
7079          * not even exist) or that it is large enough to satisfy the
7080          * requested mode.
7081          */
7082         crtc->fb = mode_fits_in_fbdev(dev, mode);
7083         if (crtc->fb == NULL) {
7084                 DRM_DEBUG_KMS("creating tmp fb for load-detection\n");
7085                 crtc->fb = intel_framebuffer_create_for_mode(dev, mode, 24, 32);
7086                 old->release_fb = crtc->fb;
7087         } else
7088                 DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
7089         if (IS_ERR(crtc->fb)) {
7090                 DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
7091                 crtc->fb = old_fb;
7092                 return false;
7093         }
7094
7095         if (!drm_crtc_helper_set_mode(crtc, mode, 0, 0, old_fb)) {
7096                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
7097                 if (old->release_fb)
7098                         old->release_fb->funcs->destroy(old->release_fb);
7099                 crtc->fb = old_fb;
7100                 return false;
7101         }
7102
7103         /* let the connector get through one full cycle before testing */
7104         intel_wait_for_vblank(dev, intel_crtc->pipe);
7105
7106         return true;
7107 }
7108
7109 void intel_release_load_detect_pipe(struct intel_encoder *intel_encoder,
7110                                     struct drm_connector *connector,
7111                                     struct intel_load_detect_pipe *old)
7112 {
7113         struct drm_encoder *encoder = &intel_encoder->base;
7114         struct drm_device *dev = encoder->dev;
7115         struct drm_crtc *crtc = encoder->crtc;
7116         struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
7117         struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
7118
7119         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
7120                       connector->base.id, drm_get_connector_name(connector),
7121                       encoder->base.id, drm_get_encoder_name(encoder));
7122
7123         if (old->load_detect_temp) {
7124                 connector->encoder = NULL;
7125                 drm_helper_disable_unused_functions(dev);
7126
7127                 if (old->release_fb)
7128                         old->release_fb->funcs->destroy(old->release_fb);
7129
7130                 return;
7131         }
7132
7133         /* Switch crtc and encoder back off if necessary */
7134         if (old->dpms_mode != DRM_MODE_DPMS_ON) {
7135                 encoder_funcs->dpms(encoder, old->dpms_mode);
7136                 crtc_funcs->dpms(crtc, old->dpms_mode);
7137         }
7138 }
7139
7140 /* Returns the clock of the currently programmed mode of the given pipe. */
7141 static int intel_crtc_clock_get(struct drm_device *dev, struct drm_crtc *crtc)
7142 {
7143         struct drm_i915_private *dev_priv = dev->dev_private;
7144         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7145         int pipe = intel_crtc->pipe;
7146         u32 dpll = I915_READ(DPLL(pipe));
7147         u32 fp;
7148         intel_clock_t clock;
7149
7150         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
7151                 fp = I915_READ(FP0(pipe));
7152         else
7153                 fp = I915_READ(FP1(pipe));
7154
7155         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
7156         if (IS_PINEVIEW(dev)) {
7157                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
7158                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
7159         } else {
7160                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
7161                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
7162         }
7163
7164         if (!IS_GEN2(dev)) {
7165                 if (IS_PINEVIEW(dev))
7166                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
7167                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
7168                 else
7169                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
7170                                DPLL_FPA01_P1_POST_DIV_SHIFT);
7171
7172                 switch (dpll & DPLL_MODE_MASK) {
7173                 case DPLLB_MODE_DAC_SERIAL:
7174                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
7175                                 5 : 10;
7176                         break;
7177                 case DPLLB_MODE_LVDS:
7178                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
7179                                 7 : 14;
7180                         break;
7181                 default:
7182                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
7183                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
7184                         return 0;
7185                 }
7186
7187                 /* XXX: Handle the 100Mhz refclk */
7188                 intel_clock(dev, 96000, &clock);
7189         } else {
7190                 bool is_lvds = (pipe == 1) && (I915_READ(LVDS) & LVDS_PORT_EN);
7191
7192                 if (is_lvds) {
7193                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
7194                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
7195                         clock.p2 = 14;
7196
7197                         if ((dpll & PLL_REF_INPUT_MASK) ==
7198                             PLLB_REF_INPUT_SPREADSPECTRUMIN) {
7199                                 /* XXX: might not be 66MHz */
7200                                 intel_clock(dev, 66000, &clock);
7201                         } else
7202                                 intel_clock(dev, 48000, &clock);
7203                 } else {
7204                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
7205                                 clock.p1 = 2;
7206                         else {
7207                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
7208                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
7209                         }
7210                         if (dpll & PLL_P2_DIVIDE_BY_4)
7211                                 clock.p2 = 4;
7212                         else
7213                                 clock.p2 = 2;
7214
7215                         intel_clock(dev, 48000, &clock);
7216                 }
7217         }
7218
7219         /* XXX: It would be nice to validate the clocks, but we can't reuse
7220          * i830PllIsValid() because it relies on the xf86_config connector
7221          * configuration being accurate, which it isn't necessarily.
7222          */
7223
7224         return clock.dot;
7225 }
7226
7227 /** Returns the currently programmed mode of the given pipe. */
7228 struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
7229                                              struct drm_crtc *crtc)
7230 {
7231         struct drm_i915_private *dev_priv = dev->dev_private;
7232         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7233         int pipe = intel_crtc->pipe;
7234         struct drm_display_mode *mode;
7235         int htot = I915_READ(HTOTAL(pipe));
7236         int hsync = I915_READ(HSYNC(pipe));
7237         int vtot = I915_READ(VTOTAL(pipe));
7238         int vsync = I915_READ(VSYNC(pipe));
7239
7240         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
7241         if (!mode)
7242                 return NULL;
7243
7244         mode->clock = intel_crtc_clock_get(dev, crtc);
7245         mode->hdisplay = (htot & 0xffff) + 1;
7246         mode->htotal = ((htot & 0xffff0000) >> 16) + 1;
7247         mode->hsync_start = (hsync & 0xffff) + 1;
7248         mode->hsync_end = ((hsync & 0xffff0000) >> 16) + 1;
7249         mode->vdisplay = (vtot & 0xffff) + 1;
7250         mode->vtotal = ((vtot & 0xffff0000) >> 16) + 1;
7251         mode->vsync_start = (vsync & 0xffff) + 1;
7252         mode->vsync_end = ((vsync & 0xffff0000) >> 16) + 1;
7253
7254         drm_mode_set_name(mode);
7255         drm_mode_set_crtcinfo(mode, 0);
7256
7257         return mode;
7258 }
7259
7260 #define GPU_IDLE_TIMEOUT 500 /* ms */
7261
7262 /* When this timer fires, we've been idle for awhile */
7263 static void intel_gpu_idle_timer(unsigned long arg)
7264 {
7265         struct drm_device *dev = (struct drm_device *)arg;
7266         drm_i915_private_t *dev_priv = dev->dev_private;
7267
7268         if (!list_empty(&dev_priv->mm.active_list)) {
7269                 /* Still processing requests, so just re-arm the timer. */
7270                 mod_timer(&dev_priv->idle_timer, jiffies +
7271                           msecs_to_jiffies(GPU_IDLE_TIMEOUT));
7272                 return;
7273         }
7274
7275         dev_priv->busy = false;
7276         queue_work(dev_priv->wq, &dev_priv->idle_work);
7277 }
7278
7279 #define CRTC_IDLE_TIMEOUT 1000 /* ms */
7280
7281 static void intel_crtc_idle_timer(unsigned long arg)
7282 {
7283         struct intel_crtc *intel_crtc = (struct intel_crtc *)arg;
7284         struct drm_crtc *crtc = &intel_crtc->base;
7285         drm_i915_private_t *dev_priv = crtc->dev->dev_private;
7286         struct intel_framebuffer *intel_fb;
7287
7288         intel_fb = to_intel_framebuffer(crtc->fb);
7289         if (intel_fb && intel_fb->obj->active) {
7290                 /* The framebuffer is still being accessed by the GPU. */
7291                 mod_timer(&intel_crtc->idle_timer, jiffies +
7292                           msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
7293                 return;
7294         }
7295
7296         intel_crtc->busy = false;
7297         queue_work(dev_priv->wq, &dev_priv->idle_work);
7298 }
7299
7300 static void intel_increase_pllclock(struct drm_crtc *crtc)
7301 {
7302         struct drm_device *dev = crtc->dev;
7303         drm_i915_private_t *dev_priv = dev->dev_private;
7304         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7305         int pipe = intel_crtc->pipe;
7306         int dpll_reg = DPLL(pipe);
7307         int dpll;
7308
7309         if (HAS_PCH_SPLIT(dev))
7310                 return;
7311
7312         if (!dev_priv->lvds_downclock_avail)
7313                 return;
7314
7315         dpll = I915_READ(dpll_reg);
7316         if (!HAS_PIPE_CXSR(dev) && (dpll & DISPLAY_RATE_SELECT_FPA1)) {
7317                 DRM_DEBUG_DRIVER("upclocking LVDS\n");
7318
7319                 assert_panel_unlocked(dev_priv, pipe);
7320
7321                 dpll &= ~DISPLAY_RATE_SELECT_FPA1;
7322                 I915_WRITE(dpll_reg, dpll);
7323                 intel_wait_for_vblank(dev, pipe);
7324
7325                 dpll = I915_READ(dpll_reg);
7326                 if (dpll & DISPLAY_RATE_SELECT_FPA1)
7327                         DRM_DEBUG_DRIVER("failed to upclock LVDS!\n");
7328         }
7329
7330         /* Schedule downclock */
7331         mod_timer(&intel_crtc->idle_timer, jiffies +
7332                   msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
7333 }
7334
7335 static void intel_decrease_pllclock(struct drm_crtc *crtc)
7336 {
7337         struct drm_device *dev = crtc->dev;
7338         drm_i915_private_t *dev_priv = dev->dev_private;
7339         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7340         int pipe = intel_crtc->pipe;
7341         int dpll_reg = DPLL(pipe);
7342         int dpll = I915_READ(dpll_reg);
7343
7344         if (HAS_PCH_SPLIT(dev))
7345                 return;
7346
7347         if (!dev_priv->lvds_downclock_avail)
7348                 return;
7349
7350         /*
7351          * Since this is called by a timer, we should never get here in
7352          * the manual case.
7353          */
7354         if (!HAS_PIPE_CXSR(dev) && intel_crtc->lowfreq_avail) {
7355                 DRM_DEBUG_DRIVER("downclocking LVDS\n");
7356
7357                 assert_panel_unlocked(dev_priv, pipe);
7358
7359                 dpll |= DISPLAY_RATE_SELECT_FPA1;
7360                 I915_WRITE(dpll_reg, dpll);
7361                 intel_wait_for_vblank(dev, pipe);
7362                 dpll = I915_READ(dpll_reg);
7363                 if (!(dpll & DISPLAY_RATE_SELECT_FPA1))
7364                         DRM_DEBUG_DRIVER("failed to downclock LVDS!\n");
7365         }
7366
7367 }
7368
7369 /**
7370  * intel_idle_update - adjust clocks for idleness
7371  * @work: work struct
7372  *
7373  * Either the GPU or display (or both) went idle.  Check the busy status
7374  * here and adjust the CRTC and GPU clocks as necessary.
7375  */
7376 static void intel_idle_update(struct work_struct *work)
7377 {
7378         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
7379                                                     idle_work);
7380         struct drm_device *dev = dev_priv->dev;
7381         struct drm_crtc *crtc;
7382         struct intel_crtc *intel_crtc;
7383
7384         if (!i915_powersave)
7385                 return;
7386
7387         mutex_lock(&dev->struct_mutex);
7388
7389         i915_update_gfx_val(dev_priv);
7390
7391         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
7392                 /* Skip inactive CRTCs */
7393                 if (!crtc->fb)
7394                         continue;
7395
7396                 intel_crtc = to_intel_crtc(crtc);
7397                 if (!intel_crtc->busy)
7398                         intel_decrease_pllclock(crtc);
7399         }
7400
7401
7402         mutex_unlock(&dev->struct_mutex);
7403 }
7404
7405 /**
7406  * intel_mark_busy - mark the GPU and possibly the display busy
7407  * @dev: drm device
7408  * @obj: object we're operating on
7409  *
7410  * Callers can use this function to indicate that the GPU is busy processing
7411  * commands.  If @obj matches one of the CRTC objects (i.e. it's a scanout
7412  * buffer), we'll also mark the display as busy, so we know to increase its
7413  * clock frequency.
7414  */
7415 void intel_mark_busy(struct drm_device *dev, struct drm_i915_gem_object *obj)
7416 {
7417         drm_i915_private_t *dev_priv = dev->dev_private;
7418         struct drm_crtc *crtc = NULL;
7419         struct intel_framebuffer *intel_fb;
7420         struct intel_crtc *intel_crtc;
7421
7422         if (!drm_core_check_feature(dev, DRIVER_MODESET))
7423                 return;
7424
7425         if (!dev_priv->busy)
7426                 dev_priv->busy = true;
7427         else
7428                 mod_timer(&dev_priv->idle_timer, jiffies +
7429                           msecs_to_jiffies(GPU_IDLE_TIMEOUT));
7430
7431         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
7432                 if (!crtc->fb)
7433                         continue;
7434
7435                 intel_crtc = to_intel_crtc(crtc);
7436                 intel_fb = to_intel_framebuffer(crtc->fb);
7437                 if (intel_fb->obj == obj) {
7438                         if (!intel_crtc->busy) {
7439                                 /* Non-busy -> busy, upclock */
7440                                 intel_increase_pllclock(crtc);
7441                                 intel_crtc->busy = true;
7442                         } else {
7443                                 /* Busy -> busy, put off timer */
7444                                 mod_timer(&intel_crtc->idle_timer, jiffies +
7445                                           msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
7446                         }
7447                 }
7448         }
7449 }
7450
7451 static void intel_crtc_destroy(struct drm_crtc *crtc)
7452 {
7453         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7454         struct drm_device *dev = crtc->dev;
7455         struct intel_unpin_work *work;
7456         unsigned long flags;
7457
7458         spin_lock_irqsave(&dev->event_lock, flags);
7459         work = intel_crtc->unpin_work;
7460         intel_crtc->unpin_work = NULL;
7461         spin_unlock_irqrestore(&dev->event_lock, flags);
7462
7463         if (work) {
7464                 cancel_work_sync(&work->work);
7465                 kfree(work);
7466         }
7467
7468         drm_crtc_cleanup(crtc);
7469
7470         kfree(intel_crtc);
7471 }
7472
7473 static void intel_unpin_work_fn(struct work_struct *__work)
7474 {
7475         struct intel_unpin_work *work =
7476                 container_of(__work, struct intel_unpin_work, work);
7477
7478         mutex_lock(&work->dev->struct_mutex);
7479         intel_unpin_fb_obj(work->old_fb_obj);
7480         drm_gem_object_unreference(&work->pending_flip_obj->base);
7481         drm_gem_object_unreference(&work->old_fb_obj->base);
7482
7483         intel_update_fbc(work->dev);
7484         mutex_unlock(&work->dev->struct_mutex);
7485         kfree(work);
7486 }
7487
7488 static void do_intel_finish_page_flip(struct drm_device *dev,
7489                                       struct drm_crtc *crtc)
7490 {
7491         drm_i915_private_t *dev_priv = dev->dev_private;
7492         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7493         struct intel_unpin_work *work;
7494         struct drm_i915_gem_object *obj;
7495         struct drm_pending_vblank_event *e;
7496         struct timeval tnow, tvbl;
7497         unsigned long flags;
7498
7499         /* Ignore early vblank irqs */
7500         if (intel_crtc == NULL)
7501                 return;
7502
7503         do_gettimeofday(&tnow);
7504
7505         spin_lock_irqsave(&dev->event_lock, flags);
7506         work = intel_crtc->unpin_work;
7507         if (work == NULL || !work->pending) {
7508                 spin_unlock_irqrestore(&dev->event_lock, flags);
7509                 return;
7510         }
7511
7512         intel_crtc->unpin_work = NULL;
7513
7514         if (work->event) {
7515                 e = work->event;
7516                 e->event.sequence = drm_vblank_count_and_time(dev, intel_crtc->pipe, &tvbl);
7517
7518                 /* Called before vblank count and timestamps have
7519                  * been updated for the vblank interval of flip
7520                  * completion? Need to increment vblank count and
7521                  * add one videorefresh duration to returned timestamp
7522                  * to account for this. We assume this happened if we
7523                  * get called over 0.9 frame durations after the last
7524                  * timestamped vblank.
7525                  *
7526                  * This calculation can not be used with vrefresh rates
7527                  * below 5Hz (10Hz to be on the safe side) without
7528                  * promoting to 64 integers.
7529                  */
7530                 if (10 * (timeval_to_ns(&tnow) - timeval_to_ns(&tvbl)) >
7531                     9 * crtc->framedur_ns) {
7532                         e->event.sequence++;
7533                         tvbl = ns_to_timeval(timeval_to_ns(&tvbl) +
7534                                              crtc->framedur_ns);
7535                 }
7536
7537                 e->event.tv_sec = tvbl.tv_sec;
7538                 e->event.tv_usec = tvbl.tv_usec;
7539
7540                 list_add_tail(&e->base.link,
7541                               &e->base.file_priv->event_list);
7542                 wake_up_interruptible(&e->base.file_priv->event_wait);
7543         }
7544
7545         drm_vblank_put(dev, intel_crtc->pipe);
7546
7547         spin_unlock_irqrestore(&dev->event_lock, flags);
7548
7549         obj = work->old_fb_obj;
7550
7551         atomic_clear_mask(1 << intel_crtc->plane,
7552                           &obj->pending_flip.counter);
7553         if (atomic_read(&obj->pending_flip) == 0)
7554                 wake_up(&dev_priv->pending_flip_queue);
7555
7556         schedule_work(&work->work);
7557
7558         trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj);
7559 }
7560
7561 void intel_finish_page_flip(struct drm_device *dev, int pipe)
7562 {
7563         drm_i915_private_t *dev_priv = dev->dev_private;
7564         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
7565
7566         do_intel_finish_page_flip(dev, crtc);
7567 }
7568
7569 void intel_finish_page_flip_plane(struct drm_device *dev, int plane)
7570 {
7571         drm_i915_private_t *dev_priv = dev->dev_private;
7572         struct drm_crtc *crtc = dev_priv->plane_to_crtc_mapping[plane];
7573
7574         do_intel_finish_page_flip(dev, crtc);
7575 }
7576
7577 void intel_prepare_page_flip(struct drm_device *dev, int plane)
7578 {
7579         drm_i915_private_t *dev_priv = dev->dev_private;
7580         struct intel_crtc *intel_crtc =
7581                 to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]);
7582         unsigned long flags;
7583
7584         spin_lock_irqsave(&dev->event_lock, flags);
7585         if (intel_crtc->unpin_work) {
7586                 if ((++intel_crtc->unpin_work->pending) > 1)
7587                         DRM_ERROR("Prepared flip multiple times\n");
7588         } else {
7589                 DRM_DEBUG_DRIVER("preparing flip with no unpin work?\n");
7590         }
7591         spin_unlock_irqrestore(&dev->event_lock, flags);
7592 }
7593
7594 static int intel_gen2_queue_flip(struct drm_device *dev,
7595                                  struct drm_crtc *crtc,
7596                                  struct drm_framebuffer *fb,
7597                                  struct drm_i915_gem_object *obj)
7598 {
7599         struct drm_i915_private *dev_priv = dev->dev_private;
7600         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7601         unsigned long offset;
7602         u32 flip_mask;
7603         int ret;
7604
7605         ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
7606         if (ret)
7607                 goto out;
7608
7609         /* Offset into the new buffer for cases of shared fbs between CRTCs */
7610         offset = crtc->y * fb->pitches[0] + crtc->x * fb->bits_per_pixel/8;
7611
7612         ret = BEGIN_LP_RING(6);
7613         if (ret)
7614                 goto out;
7615
7616         /* Can't queue multiple flips, so wait for the previous
7617          * one to finish before executing the next.
7618          */
7619         if (intel_crtc->plane)
7620                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
7621         else
7622                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
7623         OUT_RING(MI_WAIT_FOR_EVENT | flip_mask);
7624         OUT_RING(MI_NOOP);
7625         OUT_RING(MI_DISPLAY_FLIP |
7626                  MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
7627         OUT_RING(fb->pitches[0]);
7628         OUT_RING(obj->gtt_offset + offset);
7629         OUT_RING(0); /* aux display base address, unused */
7630         ADVANCE_LP_RING();
7631 out:
7632         return ret;
7633 }
7634
7635 static int intel_gen3_queue_flip(struct drm_device *dev,
7636                                  struct drm_crtc *crtc,
7637                                  struct drm_framebuffer *fb,
7638                                  struct drm_i915_gem_object *obj)
7639 {
7640         struct drm_i915_private *dev_priv = dev->dev_private;
7641         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7642         unsigned long offset;
7643         u32 flip_mask;
7644         int ret;
7645
7646         ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
7647         if (ret)
7648                 goto out;
7649
7650         /* Offset into the new buffer for cases of shared fbs between CRTCs */
7651         offset = crtc->y * fb->pitches[0] + crtc->x * fb->bits_per_pixel/8;
7652
7653         ret = BEGIN_LP_RING(6);
7654         if (ret)
7655                 goto out;
7656
7657         if (intel_crtc->plane)
7658                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
7659         else
7660                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
7661         OUT_RING(MI_WAIT_FOR_EVENT | flip_mask);
7662         OUT_RING(MI_NOOP);
7663         OUT_RING(MI_DISPLAY_FLIP_I915 |
7664                  MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
7665         OUT_RING(fb->pitches[0]);
7666         OUT_RING(obj->gtt_offset + offset);
7667         OUT_RING(MI_NOOP);
7668
7669         ADVANCE_LP_RING();
7670 out:
7671         return ret;
7672 }
7673
7674 static int intel_gen4_queue_flip(struct drm_device *dev,
7675                                  struct drm_crtc *crtc,
7676                                  struct drm_framebuffer *fb,
7677                                  struct drm_i915_gem_object *obj)
7678 {
7679         struct drm_i915_private *dev_priv = dev->dev_private;
7680         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7681         uint32_t pf, pipesrc;
7682         int ret;
7683
7684         ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
7685         if (ret)
7686                 goto out;
7687
7688         ret = BEGIN_LP_RING(4);
7689         if (ret)
7690                 goto out;
7691
7692         /* i965+ uses the linear or tiled offsets from the
7693          * Display Registers (which do not change across a page-flip)
7694          * so we need only reprogram the base address.
7695          */
7696         OUT_RING(MI_DISPLAY_FLIP |
7697                  MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
7698         OUT_RING(fb->pitches[0]);
7699         OUT_RING(obj->gtt_offset | obj->tiling_mode);
7700
7701         /* XXX Enabling the panel-fitter across page-flip is so far
7702          * untested on non-native modes, so ignore it for now.
7703          * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
7704          */
7705         pf = 0;
7706         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
7707         OUT_RING(pf | pipesrc);
7708         ADVANCE_LP_RING();
7709 out:
7710         return ret;
7711 }
7712
7713 static int intel_gen6_queue_flip(struct drm_device *dev,
7714                                  struct drm_crtc *crtc,
7715                                  struct drm_framebuffer *fb,
7716                                  struct drm_i915_gem_object *obj)
7717 {
7718         struct drm_i915_private *dev_priv = dev->dev_private;
7719         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7720         uint32_t pf, pipesrc;
7721         int ret;
7722
7723         ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
7724         if (ret)
7725                 goto out;
7726
7727         ret = BEGIN_LP_RING(4);
7728         if (ret)
7729                 goto out;
7730
7731         OUT_RING(MI_DISPLAY_FLIP |
7732                  MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
7733         OUT_RING(fb->pitches[0] | obj->tiling_mode);
7734         OUT_RING(obj->gtt_offset);
7735
7736         pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE;
7737         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
7738         OUT_RING(pf | pipesrc);
7739         ADVANCE_LP_RING();
7740 out:
7741         return ret;
7742 }
7743
7744 /*
7745  * On gen7 we currently use the blit ring because (in early silicon at least)
7746  * the render ring doesn't give us interrpts for page flip completion, which
7747  * means clients will hang after the first flip is queued.  Fortunately the
7748  * blit ring generates interrupts properly, so use it instead.
7749  */
7750 static int intel_gen7_queue_flip(struct drm_device *dev,
7751                                  struct drm_crtc *crtc,
7752                                  struct drm_framebuffer *fb,
7753                                  struct drm_i915_gem_object *obj)
7754 {
7755         struct drm_i915_private *dev_priv = dev->dev_private;
7756         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7757         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
7758         int ret;
7759
7760         ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
7761         if (ret)
7762                 goto out;
7763
7764         ret = intel_ring_begin(ring, 4);
7765         if (ret)
7766                 goto out;
7767
7768         intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | (intel_crtc->plane << 19));
7769         intel_ring_emit(ring, (fb->pitches[0] | obj->tiling_mode));
7770         intel_ring_emit(ring, (obj->gtt_offset));
7771         intel_ring_emit(ring, (MI_NOOP));
7772         intel_ring_advance(ring);
7773 out:
7774         return ret;
7775 }
7776
7777 static int intel_default_queue_flip(struct drm_device *dev,
7778                                     struct drm_crtc *crtc,
7779                                     struct drm_framebuffer *fb,
7780                                     struct drm_i915_gem_object *obj)
7781 {
7782         return -ENODEV;
7783 }
7784
7785 static int intel_crtc_page_flip(struct drm_crtc *crtc,
7786                                 struct drm_framebuffer *fb,
7787                                 struct drm_pending_vblank_event *event)
7788 {
7789         struct drm_device *dev = crtc->dev;
7790         struct drm_i915_private *dev_priv = dev->dev_private;
7791         struct intel_framebuffer *intel_fb;
7792         struct drm_i915_gem_object *obj;
7793         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7794         struct intel_unpin_work *work;
7795         unsigned long flags;
7796         int ret;
7797
7798         work = kzalloc(sizeof *work, GFP_KERNEL);
7799         if (work == NULL)
7800                 return -ENOMEM;
7801
7802         work->event = event;
7803         work->dev = crtc->dev;
7804         intel_fb = to_intel_framebuffer(crtc->fb);
7805         work->old_fb_obj = intel_fb->obj;
7806         INIT_WORK(&work->work, intel_unpin_work_fn);
7807
7808         ret = drm_vblank_get(dev, intel_crtc->pipe);
7809         if (ret)
7810                 goto free_work;
7811
7812         /* We borrow the event spin lock for protecting unpin_work */
7813         spin_lock_irqsave(&dev->event_lock, flags);
7814         if (intel_crtc->unpin_work) {
7815                 spin_unlock_irqrestore(&dev->event_lock, flags);
7816                 kfree(work);
7817                 drm_vblank_put(dev, intel_crtc->pipe);
7818
7819                 DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
7820                 return -EBUSY;
7821         }
7822         intel_crtc->unpin_work = work;
7823         spin_unlock_irqrestore(&dev->event_lock, flags);
7824
7825         intel_fb = to_intel_framebuffer(fb);
7826         obj = intel_fb->obj;
7827
7828         mutex_lock(&dev->struct_mutex);
7829
7830         /* Reference the objects for the scheduled work. */
7831         drm_gem_object_reference(&work->old_fb_obj->base);
7832         drm_gem_object_reference(&obj->base);
7833
7834         crtc->fb = fb;
7835
7836         work->pending_flip_obj = obj;
7837
7838         work->enable_stall_check = true;
7839
7840         /* Block clients from rendering to the new back buffer until
7841          * the flip occurs and the object is no longer visible.
7842          */
7843         atomic_add(1 << intel_crtc->plane, &work->old_fb_obj->pending_flip);
7844
7845         ret = dev_priv->display.queue_flip(dev, crtc, fb, obj);
7846         if (ret)
7847                 goto cleanup_pending;
7848
7849         intel_disable_fbc(dev);
7850         mutex_unlock(&dev->struct_mutex);
7851
7852         trace_i915_flip_request(intel_crtc->plane, obj);
7853
7854         return 0;
7855
7856 cleanup_pending:
7857         atomic_sub(1 << intel_crtc->plane, &work->old_fb_obj->pending_flip);
7858         drm_gem_object_unreference(&work->old_fb_obj->base);
7859         drm_gem_object_unreference(&obj->base);
7860         mutex_unlock(&dev->struct_mutex);
7861
7862         spin_lock_irqsave(&dev->event_lock, flags);
7863         intel_crtc->unpin_work = NULL;
7864         spin_unlock_irqrestore(&dev->event_lock, flags);
7865
7866         drm_vblank_put(dev, intel_crtc->pipe);
7867 free_work:
7868         kfree(work);
7869
7870         return ret;
7871 }
7872
7873 static void intel_sanitize_modesetting(struct drm_device *dev,
7874                                        int pipe, int plane)
7875 {
7876         struct drm_i915_private *dev_priv = dev->dev_private;
7877         u32 reg, val;
7878
7879         if (HAS_PCH_SPLIT(dev))
7880                 return;
7881
7882         /* Who knows what state these registers were left in by the BIOS or
7883          * grub?
7884          *
7885          * If we leave the registers in a conflicting state (e.g. with the
7886          * display plane reading from the other pipe than the one we intend
7887          * to use) then when we attempt to teardown the active mode, we will
7888          * not disable the pipes and planes in the correct order -- leaving
7889          * a plane reading from a disabled pipe and possibly leading to
7890          * undefined behaviour.
7891          */
7892
7893         reg = DSPCNTR(plane);
7894         val = I915_READ(reg);
7895
7896         if ((val & DISPLAY_PLANE_ENABLE) == 0)
7897                 return;
7898         if (!!(val & DISPPLANE_SEL_PIPE_MASK) == pipe)
7899                 return;
7900
7901         /* This display plane is active and attached to the other CPU pipe. */
7902         pipe = !pipe;
7903
7904         /* Disable the plane and wait for it to stop reading from the pipe. */
7905         intel_disable_plane(dev_priv, plane, pipe);
7906         intel_disable_pipe(dev_priv, pipe);
7907 }
7908
7909 static void intel_crtc_reset(struct drm_crtc *crtc)
7910 {
7911         struct drm_device *dev = crtc->dev;
7912         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7913
7914         /* Reset flags back to the 'unknown' status so that they
7915          * will be correctly set on the initial modeset.
7916          */
7917         intel_crtc->dpms_mode = -1;
7918
7919         /* We need to fix up any BIOS configuration that conflicts with
7920          * our expectations.
7921          */
7922         intel_sanitize_modesetting(dev, intel_crtc->pipe, intel_crtc->plane);
7923 }
7924
7925 static struct drm_crtc_helper_funcs intel_helper_funcs = {
7926         .dpms = intel_crtc_dpms,
7927         .mode_fixup = intel_crtc_mode_fixup,
7928         .mode_set = intel_crtc_mode_set,
7929         .mode_set_base = intel_pipe_set_base,
7930         .mode_set_base_atomic = intel_pipe_set_base_atomic,
7931         .load_lut = intel_crtc_load_lut,
7932         .disable = intel_crtc_disable,
7933 };
7934
7935 static const struct drm_crtc_funcs intel_crtc_funcs = {
7936         .reset = intel_crtc_reset,
7937         .cursor_set = intel_crtc_cursor_set,
7938         .cursor_move = intel_crtc_cursor_move,
7939         .gamma_set = intel_crtc_gamma_set,
7940         .set_config = drm_crtc_helper_set_config,
7941         .destroy = intel_crtc_destroy,
7942         .page_flip = intel_crtc_page_flip,
7943 };
7944
7945 static void intel_crtc_init(struct drm_device *dev, int pipe)
7946 {
7947         drm_i915_private_t *dev_priv = dev->dev_private;
7948         struct intel_crtc *intel_crtc;
7949         int i;
7950
7951         intel_crtc = kzalloc(sizeof(struct intel_crtc) + (INTELFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL);
7952         if (intel_crtc == NULL)
7953                 return;
7954
7955         drm_crtc_init(dev, &intel_crtc->base, &intel_crtc_funcs);
7956
7957         drm_mode_crtc_set_gamma_size(&intel_crtc->base, 256);
7958         for (i = 0; i < 256; i++) {
7959                 intel_crtc->lut_r[i] = i;
7960                 intel_crtc->lut_g[i] = i;
7961                 intel_crtc->lut_b[i] = i;
7962         }
7963
7964         /* Swap pipes & planes for FBC on pre-965 */
7965         intel_crtc->pipe = pipe;
7966         intel_crtc->plane = pipe;
7967         if (IS_MOBILE(dev) && IS_GEN3(dev)) {
7968                 DRM_DEBUG_KMS("swapping pipes & planes for FBC\n");
7969                 intel_crtc->plane = !pipe;
7970         }
7971
7972         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
7973                dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL);
7974         dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base;
7975         dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base;
7976
7977         intel_crtc_reset(&intel_crtc->base);
7978         intel_crtc->active = true; /* force the pipe off on setup_init_config */
7979         intel_crtc->bpp = 24; /* default for pre-Ironlake */
7980
7981         if (HAS_PCH_SPLIT(dev)) {
7982                 if (pipe == 2 && IS_IVYBRIDGE(dev))
7983                         intel_crtc->no_pll = true;
7984                 intel_helper_funcs.prepare = ironlake_crtc_prepare;
7985                 intel_helper_funcs.commit = ironlake_crtc_commit;
7986         } else {
7987                 intel_helper_funcs.prepare = i9xx_crtc_prepare;
7988                 intel_helper_funcs.commit = i9xx_crtc_commit;
7989         }
7990
7991         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
7992
7993         intel_crtc->busy = false;
7994
7995         setup_timer(&intel_crtc->idle_timer, intel_crtc_idle_timer,
7996                     (unsigned long)intel_crtc);
7997 }
7998
7999 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
8000                                 struct drm_file *file)
8001 {
8002         drm_i915_private_t *dev_priv = dev->dev_private;
8003         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
8004         struct drm_mode_object *drmmode_obj;
8005         struct intel_crtc *crtc;
8006
8007         if (!dev_priv) {
8008                 DRM_ERROR("called with no initialization\n");
8009                 return -EINVAL;
8010         }
8011
8012         drmmode_obj = drm_mode_object_find(dev, pipe_from_crtc_id->crtc_id,
8013                         DRM_MODE_OBJECT_CRTC);
8014
8015         if (!drmmode_obj) {
8016                 DRM_ERROR("no such CRTC id\n");
8017                 return -EINVAL;
8018         }
8019
8020         crtc = to_intel_crtc(obj_to_crtc(drmmode_obj));
8021         pipe_from_crtc_id->pipe = crtc->pipe;
8022
8023         return 0;
8024 }
8025
8026 static int intel_encoder_clones(struct drm_device *dev, int type_mask)
8027 {
8028         struct intel_encoder *encoder;
8029         int index_mask = 0;
8030         int entry = 0;
8031
8032         list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
8033                 if (type_mask & encoder->clone_mask)
8034                         index_mask |= (1 << entry);
8035                 entry++;
8036         }
8037
8038         return index_mask;
8039 }
8040
8041 static bool has_edp_a(struct drm_device *dev)
8042 {
8043         struct drm_i915_private *dev_priv = dev->dev_private;
8044
8045         if (!IS_MOBILE(dev))
8046                 return false;
8047
8048         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
8049                 return false;
8050
8051         if (IS_GEN5(dev) &&
8052             (I915_READ(ILK_DISPLAY_CHICKEN_FUSES) & ILK_eDP_A_DISABLE))
8053                 return false;
8054
8055         return true;
8056 }
8057
8058 static void intel_setup_outputs(struct drm_device *dev)
8059 {
8060         struct drm_i915_private *dev_priv = dev->dev_private;
8061         struct intel_encoder *encoder;
8062         bool dpd_is_edp = false;
8063         bool has_lvds;
8064
8065         has_lvds = intel_lvds_init(dev);
8066         if (!has_lvds && !HAS_PCH_SPLIT(dev)) {
8067                 /* disable the panel fitter on everything but LVDS */
8068                 I915_WRITE(PFIT_CONTROL, 0);
8069         }
8070
8071         if (HAS_PCH_SPLIT(dev)) {
8072                 dpd_is_edp = intel_dpd_is_edp(dev);
8073
8074                 if (has_edp_a(dev))
8075                         intel_dp_init(dev, DP_A);
8076
8077                 if (dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED))
8078                         intel_dp_init(dev, PCH_DP_D);
8079         }
8080
8081         intel_crt_init(dev);
8082
8083         if (HAS_PCH_SPLIT(dev)) {
8084                 int found;
8085
8086                 if (I915_READ(HDMIB) & PORT_DETECTED) {
8087                         /* PCH SDVOB multiplex with HDMIB */
8088                         found = intel_sdvo_init(dev, PCH_SDVOB, true);
8089                         if (!found)
8090                                 intel_hdmi_init(dev, HDMIB);
8091                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
8092                                 intel_dp_init(dev, PCH_DP_B);
8093                 }
8094
8095                 if (I915_READ(HDMIC) & PORT_DETECTED)
8096                         intel_hdmi_init(dev, HDMIC);
8097
8098                 if (I915_READ(HDMID) & PORT_DETECTED)
8099                         intel_hdmi_init(dev, HDMID);
8100
8101                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
8102                         intel_dp_init(dev, PCH_DP_C);
8103
8104                 if (!dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED))
8105                         intel_dp_init(dev, PCH_DP_D);
8106
8107         } else if (SUPPORTS_DIGITAL_OUTPUTS(dev)) {
8108                 bool found = false;
8109
8110                 if (I915_READ(SDVOB) & SDVO_DETECTED) {
8111                         DRM_DEBUG_KMS("probing SDVOB\n");
8112                         found = intel_sdvo_init(dev, SDVOB, true);
8113                         if (!found && SUPPORTS_INTEGRATED_HDMI(dev)) {
8114                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
8115                                 intel_hdmi_init(dev, SDVOB);
8116                         }
8117
8118                         if (!found && SUPPORTS_INTEGRATED_DP(dev)) {
8119                                 DRM_DEBUG_KMS("probing DP_B\n");
8120                                 intel_dp_init(dev, DP_B);
8121                         }
8122                 }
8123
8124                 /* Before G4X SDVOC doesn't have its own detect register */
8125
8126                 if (I915_READ(SDVOB) & SDVO_DETECTED) {
8127                         DRM_DEBUG_KMS("probing SDVOC\n");
8128                         found = intel_sdvo_init(dev, SDVOC, false);
8129                 }
8130
8131                 if (!found && (I915_READ(SDVOC) & SDVO_DETECTED)) {
8132
8133                         if (SUPPORTS_INTEGRATED_HDMI(dev)) {
8134                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
8135                                 intel_hdmi_init(dev, SDVOC);
8136                         }
8137                         if (SUPPORTS_INTEGRATED_DP(dev)) {
8138                                 DRM_DEBUG_KMS("probing DP_C\n");
8139                                 intel_dp_init(dev, DP_C);
8140                         }
8141                 }
8142
8143                 if (SUPPORTS_INTEGRATED_DP(dev) &&
8144                     (I915_READ(DP_D) & DP_DETECTED)) {
8145                         DRM_DEBUG_KMS("probing DP_D\n");
8146                         intel_dp_init(dev, DP_D);
8147                 }
8148         } else if (IS_GEN2(dev))
8149                 intel_dvo_init(dev);
8150
8151         if (SUPPORTS_TV(dev))
8152                 intel_tv_init(dev);
8153
8154         list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
8155                 encoder->base.possible_crtcs = encoder->crtc_mask;
8156                 encoder->base.possible_clones =
8157                         intel_encoder_clones(dev, encoder->clone_mask);
8158         }
8159
8160         /* disable all the possible outputs/crtcs before entering KMS mode */
8161         drm_helper_disable_unused_functions(dev);
8162
8163         if (HAS_PCH_SPLIT(dev))
8164                 ironlake_init_pch_refclk(dev);
8165 }
8166
8167 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
8168 {
8169         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
8170
8171         drm_framebuffer_cleanup(fb);
8172         drm_gem_object_unreference_unlocked(&intel_fb->obj->base);
8173
8174         kfree(intel_fb);
8175 }
8176
8177 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
8178                                                 struct drm_file *file,
8179                                                 unsigned int *handle)
8180 {
8181         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
8182         struct drm_i915_gem_object *obj = intel_fb->obj;
8183
8184         return drm_gem_handle_create(file, &obj->base, handle);
8185 }
8186
8187 static const struct drm_framebuffer_funcs intel_fb_funcs = {
8188         .destroy = intel_user_framebuffer_destroy,
8189         .create_handle = intel_user_framebuffer_create_handle,
8190 };
8191
8192 int intel_framebuffer_init(struct drm_device *dev,
8193                            struct intel_framebuffer *intel_fb,
8194                            struct drm_mode_fb_cmd2 *mode_cmd,
8195                            struct drm_i915_gem_object *obj)
8196 {
8197         int ret;
8198
8199         if (obj->tiling_mode == I915_TILING_Y)
8200                 return -EINVAL;
8201
8202         if (mode_cmd->pitches[0] & 63)
8203                 return -EINVAL;
8204
8205         switch (mode_cmd->pixel_format) {
8206         case DRM_FORMAT_RGB332:
8207         case DRM_FORMAT_RGB565:
8208         case DRM_FORMAT_XRGB8888:
8209         case DRM_FORMAT_ARGB8888:
8210         case DRM_FORMAT_XRGB2101010:
8211         case DRM_FORMAT_ARGB2101010:
8212                 /* RGB formats are common across chipsets */
8213                 break;
8214         case DRM_FORMAT_YUYV:
8215         case DRM_FORMAT_UYVY:
8216         case DRM_FORMAT_YVYU:
8217         case DRM_FORMAT_VYUY:
8218                 break;
8219         default:
8220                 DRM_DEBUG_KMS("unsupported pixel format %u\n",
8221                                 mode_cmd->pixel_format);
8222                 return -EINVAL;
8223         }
8224
8225         ret = drm_framebuffer_init(dev, &intel_fb->base, &intel_fb_funcs);
8226         if (ret) {
8227                 DRM_ERROR("framebuffer init failed %d\n", ret);
8228                 return ret;
8229         }
8230
8231         drm_helper_mode_fill_fb_struct(&intel_fb->base, mode_cmd);
8232         intel_fb->obj = obj;
8233         return 0;
8234 }
8235
8236 static struct drm_framebuffer *
8237 intel_user_framebuffer_create(struct drm_device *dev,
8238                               struct drm_file *filp,
8239                               struct drm_mode_fb_cmd2 *mode_cmd)
8240 {
8241         struct drm_i915_gem_object *obj;
8242
8243         obj = to_intel_bo(drm_gem_object_lookup(dev, filp,
8244                                                 mode_cmd->handles[0]));
8245         if (&obj->base == NULL)
8246                 return ERR_PTR(-ENOENT);
8247
8248         return intel_framebuffer_create(dev, mode_cmd, obj);
8249 }
8250
8251 static const struct drm_mode_config_funcs intel_mode_funcs = {
8252         .fb_create = intel_user_framebuffer_create,
8253         .output_poll_changed = intel_fb_output_poll_changed,
8254 };
8255
8256 static struct drm_i915_gem_object *
8257 intel_alloc_context_page(struct drm_device *dev)
8258 {
8259         struct drm_i915_gem_object *ctx;
8260         int ret;
8261
8262         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
8263
8264         ctx = i915_gem_alloc_object(dev, 4096);
8265         if (!ctx) {
8266                 DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
8267                 return NULL;
8268         }
8269
8270         ret = i915_gem_object_pin(ctx, 4096, true);
8271         if (ret) {
8272                 DRM_ERROR("failed to pin power context: %d\n", ret);
8273                 goto err_unref;
8274         }
8275
8276         ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
8277         if (ret) {
8278                 DRM_ERROR("failed to set-domain on power context: %d\n", ret);
8279                 goto err_unpin;
8280         }
8281
8282         return ctx;
8283
8284 err_unpin:
8285         i915_gem_object_unpin(ctx);
8286 err_unref:
8287         drm_gem_object_unreference(&ctx->base);
8288         mutex_unlock(&dev->struct_mutex);
8289         return NULL;
8290 }
8291
8292 bool ironlake_set_drps(struct drm_device *dev, u8 val)
8293 {
8294         struct drm_i915_private *dev_priv = dev->dev_private;
8295         u16 rgvswctl;
8296
8297         rgvswctl = I915_READ16(MEMSWCTL);
8298         if (rgvswctl & MEMCTL_CMD_STS) {
8299                 DRM_DEBUG("gpu busy, RCS change rejected\n");
8300                 return false; /* still busy with another command */
8301         }
8302
8303         rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
8304                 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
8305         I915_WRITE16(MEMSWCTL, rgvswctl);
8306         POSTING_READ16(MEMSWCTL);
8307
8308         rgvswctl |= MEMCTL_CMD_STS;
8309         I915_WRITE16(MEMSWCTL, rgvswctl);
8310
8311         return true;
8312 }
8313
8314 void ironlake_enable_drps(struct drm_device *dev)
8315 {
8316         struct drm_i915_private *dev_priv = dev->dev_private;
8317         u32 rgvmodectl = I915_READ(MEMMODECTL);
8318         u8 fmax, fmin, fstart, vstart;
8319
8320         /* Enable temp reporting */
8321         I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
8322         I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
8323
8324         /* 100ms RC evaluation intervals */
8325         I915_WRITE(RCUPEI, 100000);
8326         I915_WRITE(RCDNEI, 100000);
8327
8328         /* Set max/min thresholds to 90ms and 80ms respectively */
8329         I915_WRITE(RCBMAXAVG, 90000);
8330         I915_WRITE(RCBMINAVG, 80000);
8331
8332         I915_WRITE(MEMIHYST, 1);
8333
8334         /* Set up min, max, and cur for interrupt handling */
8335         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
8336         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
8337         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
8338                 MEMMODE_FSTART_SHIFT;
8339
8340         vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
8341                 PXVFREQ_PX_SHIFT;
8342
8343         dev_priv->fmax = fmax; /* IPS callback will increase this */
8344         dev_priv->fstart = fstart;
8345
8346         dev_priv->max_delay = fstart;
8347         dev_priv->min_delay = fmin;
8348         dev_priv->cur_delay = fstart;
8349
8350         DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
8351                          fmax, fmin, fstart);
8352
8353         I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
8354
8355         /*
8356          * Interrupts will be enabled in ironlake_irq_postinstall
8357          */
8358
8359         I915_WRITE(VIDSTART, vstart);
8360         POSTING_READ(VIDSTART);
8361
8362         rgvmodectl |= MEMMODE_SWMODE_EN;
8363         I915_WRITE(MEMMODECTL, rgvmodectl);
8364
8365         if (wait_for((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
8366                 DRM_ERROR("stuck trying to change perf mode\n");
8367         msleep(1);
8368
8369         ironlake_set_drps(dev, fstart);
8370
8371         dev_priv->last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
8372                 I915_READ(0x112e0);
8373         dev_priv->last_time1 = jiffies_to_msecs(jiffies);
8374         dev_priv->last_count2 = I915_READ(0x112f4);
8375         getrawmonotonic(&dev_priv->last_time2);
8376 }
8377
8378 void ironlake_disable_drps(struct drm_device *dev)
8379 {
8380         struct drm_i915_private *dev_priv = dev->dev_private;
8381         u16 rgvswctl = I915_READ16(MEMSWCTL);
8382
8383         /* Ack interrupts, disable EFC interrupt */
8384         I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
8385         I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
8386         I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
8387         I915_WRITE(DEIIR, DE_PCU_EVENT);
8388         I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
8389
8390         /* Go back to the starting frequency */
8391         ironlake_set_drps(dev, dev_priv->fstart);
8392         msleep(1);
8393         rgvswctl |= MEMCTL_CMD_STS;
8394         I915_WRITE(MEMSWCTL, rgvswctl);
8395         msleep(1);
8396
8397 }
8398
8399 void gen6_set_rps(struct drm_device *dev, u8 val)
8400 {
8401         struct drm_i915_private *dev_priv = dev->dev_private;
8402         u32 swreq;
8403
8404         swreq = (val & 0x3ff) << 25;
8405         I915_WRITE(GEN6_RPNSWREQ, swreq);
8406 }
8407
8408 void gen6_disable_rps(struct drm_device *dev)
8409 {
8410         struct drm_i915_private *dev_priv = dev->dev_private;
8411
8412         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
8413         I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
8414         I915_WRITE(GEN6_PMIER, 0);
8415         /* Complete PM interrupt masking here doesn't race with the rps work
8416          * item again unmasking PM interrupts because that is using a different
8417          * register (PMIMR) to mask PM interrupts. The only risk is in leaving
8418          * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
8419
8420         spin_lock_irq(&dev_priv->rps_lock);
8421         dev_priv->pm_iir = 0;
8422         spin_unlock_irq(&dev_priv->rps_lock);
8423
8424         I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
8425 }
8426
8427 static unsigned long intel_pxfreq(u32 vidfreq)
8428 {
8429         unsigned long freq;
8430         int div = (vidfreq & 0x3f0000) >> 16;
8431         int post = (vidfreq & 0x3000) >> 12;
8432         int pre = (vidfreq & 0x7);
8433
8434         if (!pre)
8435                 return 0;
8436
8437         freq = ((div * 133333) / ((1<<post) * pre));
8438
8439         return freq;
8440 }
8441
8442 void intel_init_emon(struct drm_device *dev)
8443 {
8444         struct drm_i915_private *dev_priv = dev->dev_private;
8445         u32 lcfuse;
8446         u8 pxw[16];
8447         int i;
8448
8449         /* Disable to program */
8450         I915_WRITE(ECR, 0);
8451         POSTING_READ(ECR);
8452
8453         /* Program energy weights for various events */
8454         I915_WRITE(SDEW, 0x15040d00);
8455         I915_WRITE(CSIEW0, 0x007f0000);
8456         I915_WRITE(CSIEW1, 0x1e220004);
8457         I915_WRITE(CSIEW2, 0x04000004);
8458
8459         for (i = 0; i < 5; i++)
8460                 I915_WRITE(PEW + (i * 4), 0);
8461         for (i = 0; i < 3; i++)
8462                 I915_WRITE(DEW + (i * 4), 0);
8463
8464         /* Program P-state weights to account for frequency power adjustment */
8465         for (i = 0; i < 16; i++) {
8466                 u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
8467                 unsigned long freq = intel_pxfreq(pxvidfreq);
8468                 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
8469                         PXVFREQ_PX_SHIFT;
8470                 unsigned long val;
8471
8472                 val = vid * vid;
8473                 val *= (freq / 1000);
8474                 val *= 255;
8475                 val /= (127*127*900);
8476                 if (val > 0xff)
8477                         DRM_ERROR("bad pxval: %ld\n", val);
8478                 pxw[i] = val;
8479         }
8480         /* Render standby states get 0 weight */
8481         pxw[14] = 0;
8482         pxw[15] = 0;
8483
8484         for (i = 0; i < 4; i++) {
8485                 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
8486                         (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
8487                 I915_WRITE(PXW + (i * 4), val);
8488         }
8489
8490         /* Adjust magic regs to magic values (more experimental results) */
8491         I915_WRITE(OGW0, 0);
8492         I915_WRITE(OGW1, 0);
8493         I915_WRITE(EG0, 0x00007f00);
8494         I915_WRITE(EG1, 0x0000000e);
8495         I915_WRITE(EG2, 0x000e0000);
8496         I915_WRITE(EG3, 0x68000300);
8497         I915_WRITE(EG4, 0x42000000);
8498         I915_WRITE(EG5, 0x00140031);
8499         I915_WRITE(EG6, 0);
8500         I915_WRITE(EG7, 0);
8501
8502         for (i = 0; i < 8; i++)
8503                 I915_WRITE(PXWL + (i * 4), 0);
8504
8505         /* Enable PMON + select events */
8506         I915_WRITE(ECR, 0x80000019);
8507
8508         lcfuse = I915_READ(LCFUSE02);
8509
8510         dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK);
8511 }
8512
8513 static bool intel_enable_rc6(struct drm_device *dev)
8514 {
8515         /*
8516          * Respect the kernel parameter if it is set
8517          */
8518         if (i915_enable_rc6 >= 0)
8519                 return i915_enable_rc6;
8520
8521         /*
8522          * Disable RC6 on Ironlake
8523          */
8524         if (INTEL_INFO(dev)->gen == 5)
8525                 return 0;
8526
8527         /*
8528          * Disable rc6 on Sandybridge
8529          */
8530         if (INTEL_INFO(dev)->gen == 6) {
8531                 DRM_DEBUG_DRIVER("Sandybridge: RC6 disabled\n");
8532                 return 0;
8533         }
8534         DRM_DEBUG_DRIVER("RC6 enabled\n");
8535         return 1;
8536 }
8537
8538 void gen6_enable_rps(struct drm_i915_private *dev_priv)
8539 {
8540         u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
8541         u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
8542         u32 pcu_mbox, rc6_mask = 0;
8543         u32 gtfifodbg;
8544         int cur_freq, min_freq, max_freq;
8545         int i;
8546
8547         /* Here begins a magic sequence of register writes to enable
8548          * auto-downclocking.
8549          *
8550          * Perhaps there might be some value in exposing these to
8551          * userspace...
8552          */
8553         I915_WRITE(GEN6_RC_STATE, 0);
8554         mutex_lock(&dev_priv->dev->struct_mutex);
8555
8556         /* Clear the DBG now so we don't confuse earlier errors */
8557         if ((gtfifodbg = I915_READ(GTFIFODBG))) {
8558                 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
8559                 I915_WRITE(GTFIFODBG, gtfifodbg);
8560         }
8561
8562         gen6_gt_force_wake_get(dev_priv);
8563
8564         /* disable the counters and set deterministic thresholds */
8565         I915_WRITE(GEN6_RC_CONTROL, 0);
8566
8567         I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
8568         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
8569         I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
8570         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
8571         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
8572
8573         for (i = 0; i < I915_NUM_RINGS; i++)
8574                 I915_WRITE(RING_MAX_IDLE(dev_priv->ring[i].mmio_base), 10);
8575
8576         I915_WRITE(GEN6_RC_SLEEP, 0);
8577         I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
8578         I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
8579         I915_WRITE(GEN6_RC6p_THRESHOLD, 100000);
8580         I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
8581
8582         if (intel_enable_rc6(dev_priv->dev))
8583                 rc6_mask = GEN6_RC_CTL_RC6p_ENABLE |
8584                         GEN6_RC_CTL_RC6_ENABLE;
8585
8586         I915_WRITE(GEN6_RC_CONTROL,
8587                    rc6_mask |
8588                    GEN6_RC_CTL_EI_MODE(1) |
8589                    GEN6_RC_CTL_HW_ENABLE);
8590
8591         I915_WRITE(GEN6_RPNSWREQ,
8592                    GEN6_FREQUENCY(10) |
8593                    GEN6_OFFSET(0) |
8594                    GEN6_AGGRESSIVE_TURBO);
8595         I915_WRITE(GEN6_RC_VIDEO_FREQ,
8596                    GEN6_FREQUENCY(12));
8597
8598         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
8599         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
8600                    18 << 24 |
8601                    6 << 16);
8602         I915_WRITE(GEN6_RP_UP_THRESHOLD, 10000);
8603         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 1000000);
8604         I915_WRITE(GEN6_RP_UP_EI, 100000);
8605         I915_WRITE(GEN6_RP_DOWN_EI, 5000000);
8606         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
8607         I915_WRITE(GEN6_RP_CONTROL,
8608                    GEN6_RP_MEDIA_TURBO |
8609                    GEN6_RP_MEDIA_HW_MODE |
8610                    GEN6_RP_MEDIA_IS_GFX |
8611                    GEN6_RP_ENABLE |
8612                    GEN6_RP_UP_BUSY_AVG |
8613                    GEN6_RP_DOWN_IDLE_CONT);
8614
8615         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
8616                      500))
8617                 DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
8618
8619         I915_WRITE(GEN6_PCODE_DATA, 0);
8620         I915_WRITE(GEN6_PCODE_MAILBOX,
8621                    GEN6_PCODE_READY |
8622                    GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
8623         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
8624                      500))
8625                 DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
8626
8627         min_freq = (rp_state_cap & 0xff0000) >> 16;
8628         max_freq = rp_state_cap & 0xff;
8629         cur_freq = (gt_perf_status & 0xff00) >> 8;
8630
8631         /* Check for overclock support */
8632         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
8633                      500))
8634                 DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
8635         I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_READ_OC_PARAMS);
8636         pcu_mbox = I915_READ(GEN6_PCODE_DATA);
8637         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
8638                      500))
8639                 DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
8640         if (pcu_mbox & (1<<31)) { /* OC supported */
8641                 max_freq = pcu_mbox & 0xff;
8642                 DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
8643         }
8644
8645         /* In units of 100MHz */
8646         dev_priv->max_delay = max_freq;
8647         dev_priv->min_delay = min_freq;
8648         dev_priv->cur_delay = cur_freq;
8649
8650         /* requires MSI enabled */
8651         I915_WRITE(GEN6_PMIER,
8652                    GEN6_PM_MBOX_EVENT |
8653                    GEN6_PM_THERMAL_EVENT |
8654                    GEN6_PM_RP_DOWN_TIMEOUT |
8655                    GEN6_PM_RP_UP_THRESHOLD |
8656                    GEN6_PM_RP_DOWN_THRESHOLD |
8657                    GEN6_PM_RP_UP_EI_EXPIRED |
8658                    GEN6_PM_RP_DOWN_EI_EXPIRED);
8659         spin_lock_irq(&dev_priv->rps_lock);
8660         WARN_ON(dev_priv->pm_iir != 0);
8661         I915_WRITE(GEN6_PMIMR, 0);
8662         spin_unlock_irq(&dev_priv->rps_lock);
8663         /* enable all PM interrupts */
8664         I915_WRITE(GEN6_PMINTRMSK, 0);
8665
8666         gen6_gt_force_wake_put(dev_priv);
8667         mutex_unlock(&dev_priv->dev->struct_mutex);
8668 }
8669
8670 void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
8671 {
8672         int min_freq = 15;
8673         int gpu_freq, ia_freq, max_ia_freq;
8674         int scaling_factor = 180;
8675
8676         max_ia_freq = cpufreq_quick_get_max(0);
8677         /*
8678          * Default to measured freq if none found, PCU will ensure we don't go
8679          * over
8680          */
8681         if (!max_ia_freq)
8682                 max_ia_freq = tsc_khz;
8683
8684         /* Convert from kHz to MHz */
8685         max_ia_freq /= 1000;
8686
8687         mutex_lock(&dev_priv->dev->struct_mutex);
8688
8689         /*
8690          * For each potential GPU frequency, load a ring frequency we'd like
8691          * to use for memory access.  We do this by specifying the IA frequency
8692          * the PCU should use as a reference to determine the ring frequency.
8693          */
8694         for (gpu_freq = dev_priv->max_delay; gpu_freq >= dev_priv->min_delay;
8695              gpu_freq--) {
8696                 int diff = dev_priv->max_delay - gpu_freq;
8697
8698                 /*
8699                  * For GPU frequencies less than 750MHz, just use the lowest
8700                  * ring freq.
8701                  */
8702                 if (gpu_freq < min_freq)
8703                         ia_freq = 800;
8704                 else
8705                         ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
8706                 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
8707
8708                 I915_WRITE(GEN6_PCODE_DATA,
8709                            (ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT) |
8710                            gpu_freq);
8711                 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
8712                            GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
8713                 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) &
8714                               GEN6_PCODE_READY) == 0, 10)) {
8715                         DRM_ERROR("pcode write of freq table timed out\n");
8716                         continue;
8717                 }
8718         }
8719
8720         mutex_unlock(&dev_priv->dev->struct_mutex);
8721 }
8722
8723 static void ironlake_init_clock_gating(struct drm_device *dev)
8724 {
8725         struct drm_i915_private *dev_priv = dev->dev_private;
8726         uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
8727
8728         /* Required for FBC */
8729         dspclk_gate |= DPFCUNIT_CLOCK_GATE_DISABLE |
8730                 DPFCRUNIT_CLOCK_GATE_DISABLE |
8731                 DPFDUNIT_CLOCK_GATE_DISABLE;
8732         /* Required for CxSR */
8733         dspclk_gate |= DPARBUNIT_CLOCK_GATE_DISABLE;
8734
8735         I915_WRITE(PCH_3DCGDIS0,
8736                    MARIUNIT_CLOCK_GATE_DISABLE |
8737                    SVSMUNIT_CLOCK_GATE_DISABLE);
8738         I915_WRITE(PCH_3DCGDIS1,
8739                    VFMUNIT_CLOCK_GATE_DISABLE);
8740
8741         I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
8742
8743         /*
8744          * According to the spec the following bits should be set in
8745          * order to enable memory self-refresh
8746          * The bit 22/21 of 0x42004
8747          * The bit 5 of 0x42020
8748          * The bit 15 of 0x45000
8749          */
8750         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8751                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
8752                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
8753         I915_WRITE(ILK_DSPCLK_GATE,
8754                    (I915_READ(ILK_DSPCLK_GATE) |
8755                     ILK_DPARB_CLK_GATE));
8756         I915_WRITE(DISP_ARB_CTL,
8757                    (I915_READ(DISP_ARB_CTL) |
8758                     DISP_FBC_WM_DIS));
8759         I915_WRITE(WM3_LP_ILK, 0);
8760         I915_WRITE(WM2_LP_ILK, 0);
8761         I915_WRITE(WM1_LP_ILK, 0);
8762
8763         /*
8764          * Based on the document from hardware guys the following bits
8765          * should be set unconditionally in order to enable FBC.
8766          * The bit 22 of 0x42000
8767          * The bit 22 of 0x42004
8768          * The bit 7,8,9 of 0x42020.
8769          */
8770         if (IS_IRONLAKE_M(dev)) {
8771                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
8772                            I915_READ(ILK_DISPLAY_CHICKEN1) |
8773                            ILK_FBCQ_DIS);
8774                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8775                            I915_READ(ILK_DISPLAY_CHICKEN2) |
8776                            ILK_DPARB_GATE);
8777                 I915_WRITE(ILK_DSPCLK_GATE,
8778                            I915_READ(ILK_DSPCLK_GATE) |
8779                            ILK_DPFC_DIS1 |
8780                            ILK_DPFC_DIS2 |
8781                            ILK_CLK_FBC);
8782         }
8783
8784         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8785                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8786                    ILK_ELPIN_409_SELECT);
8787         I915_WRITE(_3D_CHICKEN2,
8788                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
8789                    _3D_CHICKEN2_WM_READ_PIPELINED);
8790 }
8791
8792 static void gen6_init_clock_gating(struct drm_device *dev)
8793 {
8794         struct drm_i915_private *dev_priv = dev->dev_private;
8795         int pipe;
8796         uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
8797
8798         I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
8799
8800         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8801                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8802                    ILK_ELPIN_409_SELECT);
8803
8804         I915_WRITE(WM3_LP_ILK, 0);
8805         I915_WRITE(WM2_LP_ILK, 0);
8806         I915_WRITE(WM1_LP_ILK, 0);
8807
8808         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
8809          * gating disable must be set.  Failure to set it results in
8810          * flickering pixels due to Z write ordering failures after
8811          * some amount of runtime in the Mesa "fire" demo, and Unigine
8812          * Sanctuary and Tropics, and apparently anything else with
8813          * alpha test or pixel discard.
8814          *
8815          * According to the spec, bit 11 (RCCUNIT) must also be set,
8816          * but we didn't debug actual testcases to find it out.
8817          */
8818         I915_WRITE(GEN6_UCGCTL2,
8819                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
8820                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
8821
8822         /*
8823          * According to the spec the following bits should be
8824          * set in order to enable memory self-refresh and fbc:
8825          * The bit21 and bit22 of 0x42000
8826          * The bit21 and bit22 of 0x42004
8827          * The bit5 and bit7 of 0x42020
8828          * The bit14 of 0x70180
8829          * The bit14 of 0x71180
8830          */
8831         I915_WRITE(ILK_DISPLAY_CHICKEN1,
8832                    I915_READ(ILK_DISPLAY_CHICKEN1) |
8833                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
8834         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8835                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8836                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
8837         I915_WRITE(ILK_DSPCLK_GATE,
8838                    I915_READ(ILK_DSPCLK_GATE) |
8839                    ILK_DPARB_CLK_GATE  |
8840                    ILK_DPFD_CLK_GATE);
8841
8842         for_each_pipe(pipe) {
8843                 I915_WRITE(DSPCNTR(pipe),
8844                            I915_READ(DSPCNTR(pipe)) |
8845                            DISPPLANE_TRICKLE_FEED_DISABLE);
8846                 intel_flush_display_plane(dev_priv, pipe);
8847         }
8848 }
8849
8850 static void ivybridge_init_clock_gating(struct drm_device *dev)
8851 {
8852         struct drm_i915_private *dev_priv = dev->dev_private;
8853         int pipe;
8854         uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
8855
8856         I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
8857
8858         I915_WRITE(WM3_LP_ILK, 0);
8859         I915_WRITE(WM2_LP_ILK, 0);
8860         I915_WRITE(WM1_LP_ILK, 0);
8861
8862         I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
8863
8864         I915_WRITE(IVB_CHICKEN3,
8865                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
8866                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
8867
8868         for_each_pipe(pipe) {
8869                 I915_WRITE(DSPCNTR(pipe),
8870                            I915_READ(DSPCNTR(pipe)) |
8871                            DISPPLANE_TRICKLE_FEED_DISABLE);
8872                 intel_flush_display_plane(dev_priv, pipe);
8873         }
8874 }
8875
8876 static void valleyview_init_clock_gating(struct drm_device *dev)
8877 {
8878         struct drm_i915_private *dev_priv = dev->dev_private;
8879         int pipe;
8880         uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
8881
8882         I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
8883
8884         I915_WRITE(WM3_LP_ILK, 0);
8885         I915_WRITE(WM2_LP_ILK, 0);
8886         I915_WRITE(WM1_LP_ILK, 0);
8887
8888         /* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
8889          * This implements the WaDisableRCZUnitClockGating workaround.
8890          */
8891         I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
8892
8893         I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
8894
8895         I915_WRITE(IVB_CHICKEN3,
8896                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
8897                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
8898
8899         /* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
8900         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
8901                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
8902
8903         /* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
8904         I915_WRITE(GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
8905         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
8906
8907         /* This is required by WaCatErrorRejectionIssue */
8908         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
8909                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
8910                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
8911
8912         for_each_pipe(pipe) {
8913                 I915_WRITE(DSPCNTR(pipe),
8914                            I915_READ(DSPCNTR(pipe)) |
8915                            DISPPLANE_TRICKLE_FEED_DISABLE);
8916                 intel_flush_display_plane(dev_priv, pipe);
8917         }
8918
8919         I915_WRITE(CACHE_MODE_1, I915_READ(CACHE_MODE_1) |
8920                    (PIXEL_SUBSPAN_COLLECT_OPT_DISABLE << 16) |
8921                    PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
8922 }
8923
8924 static void g4x_init_clock_gating(struct drm_device *dev)
8925 {
8926         struct drm_i915_private *dev_priv = dev->dev_private;
8927         uint32_t dspclk_gate;
8928
8929         I915_WRITE(RENCLK_GATE_D1, 0);
8930         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
8931                    GS_UNIT_CLOCK_GATE_DISABLE |
8932                    CL_UNIT_CLOCK_GATE_DISABLE);
8933         I915_WRITE(RAMCLK_GATE_D, 0);
8934         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
8935                 OVRUNIT_CLOCK_GATE_DISABLE |
8936                 OVCUNIT_CLOCK_GATE_DISABLE;
8937         if (IS_GM45(dev))
8938                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
8939         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
8940 }
8941
8942 static void crestline_init_clock_gating(struct drm_device *dev)
8943 {
8944         struct drm_i915_private *dev_priv = dev->dev_private;
8945
8946         I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
8947         I915_WRITE(RENCLK_GATE_D2, 0);
8948         I915_WRITE(DSPCLK_GATE_D, 0);
8949         I915_WRITE(RAMCLK_GATE_D, 0);
8950         I915_WRITE16(DEUC, 0);
8951 }
8952
8953 static void broadwater_init_clock_gating(struct drm_device *dev)
8954 {
8955         struct drm_i915_private *dev_priv = dev->dev_private;
8956
8957         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
8958                    I965_RCC_CLOCK_GATE_DISABLE |
8959                    I965_RCPB_CLOCK_GATE_DISABLE |
8960                    I965_ISC_CLOCK_GATE_DISABLE |
8961                    I965_FBC_CLOCK_GATE_DISABLE);
8962         I915_WRITE(RENCLK_GATE_D2, 0);
8963 }
8964
8965 static void gen3_init_clock_gating(struct drm_device *dev)
8966 {
8967         struct drm_i915_private *dev_priv = dev->dev_private;
8968         u32 dstate = I915_READ(D_STATE);
8969
8970         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
8971                 DSTATE_DOT_CLOCK_GATING;
8972         I915_WRITE(D_STATE, dstate);
8973 }
8974
8975 static void i85x_init_clock_gating(struct drm_device *dev)
8976 {
8977         struct drm_i915_private *dev_priv = dev->dev_private;
8978
8979         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
8980 }
8981
8982 static void i830_init_clock_gating(struct drm_device *dev)
8983 {
8984         struct drm_i915_private *dev_priv = dev->dev_private;
8985
8986         I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
8987 }
8988
8989 static void ibx_init_clock_gating(struct drm_device *dev)
8990 {
8991         struct drm_i915_private *dev_priv = dev->dev_private;
8992
8993         /*
8994          * On Ibex Peak and Cougar Point, we need to disable clock
8995          * gating for the panel power sequencer or it will fail to
8996          * start up when no ports are active.
8997          */
8998         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
8999 }
9000
9001 static void cpt_init_clock_gating(struct drm_device *dev)
9002 {
9003         struct drm_i915_private *dev_priv = dev->dev_private;
9004         int pipe;
9005
9006         /*
9007          * On Ibex Peak and Cougar Point, we need to disable clock
9008          * gating for the panel power sequencer or it will fail to
9009          * start up when no ports are active.
9010          */
9011         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
9012         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
9013                    DPLS_EDP_PPS_FIX_DIS);
9014         /* Without this, mode sets may fail silently on FDI */
9015         for_each_pipe(pipe)
9016                 I915_WRITE(TRANS_CHICKEN2(pipe), TRANS_AUTOTRAIN_GEN_STALL_DIS);
9017 }
9018
9019 static void ironlake_teardown_rc6(struct drm_device *dev)
9020 {
9021         struct drm_i915_private *dev_priv = dev->dev_private;
9022
9023         if (dev_priv->renderctx) {
9024                 i915_gem_object_unpin(dev_priv->renderctx);
9025                 drm_gem_object_unreference(&dev_priv->renderctx->base);
9026                 dev_priv->renderctx = NULL;
9027         }
9028
9029         if (dev_priv->pwrctx) {
9030                 i915_gem_object_unpin(dev_priv->pwrctx);
9031                 drm_gem_object_unreference(&dev_priv->pwrctx->base);
9032                 dev_priv->pwrctx = NULL;
9033         }
9034 }
9035
9036 static void ironlake_disable_rc6(struct drm_device *dev)
9037 {
9038         struct drm_i915_private *dev_priv = dev->dev_private;
9039
9040         if (I915_READ(PWRCTXA)) {
9041                 /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
9042                 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT);
9043                 wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
9044                          50);
9045
9046                 I915_WRITE(PWRCTXA, 0);
9047                 POSTING_READ(PWRCTXA);
9048
9049                 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
9050                 POSTING_READ(RSTDBYCTL);
9051         }
9052
9053         ironlake_teardown_rc6(dev);
9054 }
9055
9056 static int ironlake_setup_rc6(struct drm_device *dev)
9057 {
9058         struct drm_i915_private *dev_priv = dev->dev_private;
9059
9060         if (dev_priv->renderctx == NULL)
9061                 dev_priv->renderctx = intel_alloc_context_page(dev);
9062         if (!dev_priv->renderctx)
9063                 return -ENOMEM;
9064
9065         if (dev_priv->pwrctx == NULL)
9066                 dev_priv->pwrctx = intel_alloc_context_page(dev);
9067         if (!dev_priv->pwrctx) {
9068                 ironlake_teardown_rc6(dev);
9069                 return -ENOMEM;
9070         }
9071
9072         return 0;
9073 }
9074
9075 void ironlake_enable_rc6(struct drm_device *dev)
9076 {
9077         struct drm_i915_private *dev_priv = dev->dev_private;
9078         int ret;
9079
9080         /* rc6 disabled by default due to repeated reports of hanging during
9081          * boot and resume.
9082          */
9083         if (!intel_enable_rc6(dev))
9084                 return;
9085
9086         mutex_lock(&dev->struct_mutex);
9087         ret = ironlake_setup_rc6(dev);
9088         if (ret) {
9089                 mutex_unlock(&dev->struct_mutex);
9090                 return;
9091         }
9092
9093         /*
9094          * GPU can automatically power down the render unit if given a page
9095          * to save state.
9096          */
9097         ret = BEGIN_LP_RING(6);
9098         if (ret) {
9099                 ironlake_teardown_rc6(dev);
9100                 mutex_unlock(&dev->struct_mutex);
9101                 return;
9102         }
9103
9104         OUT_RING(MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
9105         OUT_RING(MI_SET_CONTEXT);
9106         OUT_RING(dev_priv->renderctx->gtt_offset |
9107                  MI_MM_SPACE_GTT |
9108                  MI_SAVE_EXT_STATE_EN |
9109                  MI_RESTORE_EXT_STATE_EN |
9110                  MI_RESTORE_INHIBIT);
9111         OUT_RING(MI_SUSPEND_FLUSH);
9112         OUT_RING(MI_NOOP);
9113         OUT_RING(MI_FLUSH);
9114         ADVANCE_LP_RING();
9115
9116         /*
9117          * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
9118          * does an implicit flush, combined with MI_FLUSH above, it should be
9119          * safe to assume that renderctx is valid
9120          */
9121         ret = intel_wait_ring_idle(LP_RING(dev_priv));
9122         if (ret) {
9123                 DRM_ERROR("failed to enable ironlake power power savings\n");
9124                 ironlake_teardown_rc6(dev);
9125                 mutex_unlock(&dev->struct_mutex);
9126                 return;
9127         }
9128
9129         I915_WRITE(PWRCTXA, dev_priv->pwrctx->gtt_offset | PWRCTX_EN);
9130         I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
9131         mutex_unlock(&dev->struct_mutex);
9132 }
9133
9134 void intel_init_clock_gating(struct drm_device *dev)
9135 {
9136         struct drm_i915_private *dev_priv = dev->dev_private;
9137
9138         dev_priv->display.init_clock_gating(dev);
9139
9140         if (dev_priv->display.init_pch_clock_gating)
9141                 dev_priv->display.init_pch_clock_gating(dev);
9142 }
9143
9144 /* Set up chip specific display functions */
9145 static void intel_init_display(struct drm_device *dev)
9146 {
9147         struct drm_i915_private *dev_priv = dev->dev_private;
9148
9149         /* We always want a DPMS function */
9150         if (HAS_PCH_SPLIT(dev)) {
9151                 dev_priv->display.dpms = ironlake_crtc_dpms;
9152                 dev_priv->display.crtc_mode_set = ironlake_crtc_mode_set;
9153                 dev_priv->display.update_plane = ironlake_update_plane;
9154         } else {
9155                 dev_priv->display.dpms = i9xx_crtc_dpms;
9156                 dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
9157                 dev_priv->display.update_plane = i9xx_update_plane;
9158         }
9159
9160         if (I915_HAS_FBC(dev)) {
9161                 if (HAS_PCH_SPLIT(dev)) {
9162                         dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
9163                         dev_priv->display.enable_fbc = ironlake_enable_fbc;
9164                         dev_priv->display.disable_fbc = ironlake_disable_fbc;
9165                 } else if (IS_GM45(dev)) {
9166                         dev_priv->display.fbc_enabled = g4x_fbc_enabled;
9167                         dev_priv->display.enable_fbc = g4x_enable_fbc;
9168                         dev_priv->display.disable_fbc = g4x_disable_fbc;
9169                 } else if (IS_CRESTLINE(dev)) {
9170                         dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
9171                         dev_priv->display.enable_fbc = i8xx_enable_fbc;
9172                         dev_priv->display.disable_fbc = i8xx_disable_fbc;
9173                 }
9174                 /* 855GM needs testing */
9175         }
9176
9177         /* Returns the core display clock speed */
9178         if (IS_VALLEYVIEW(dev))
9179                 dev_priv->display.get_display_clock_speed =
9180                         valleyview_get_display_clock_speed;
9181         else if (IS_I945G(dev) || (IS_G33(dev) && !IS_PINEVIEW_M(dev)))
9182                 dev_priv->display.get_display_clock_speed =
9183                         i945_get_display_clock_speed;
9184         else if (IS_I915G(dev))
9185                 dev_priv->display.get_display_clock_speed =
9186                         i915_get_display_clock_speed;
9187         else if (IS_I945GM(dev) || IS_845G(dev) || IS_PINEVIEW_M(dev))
9188                 dev_priv->display.get_display_clock_speed =
9189                         i9xx_misc_get_display_clock_speed;
9190         else if (IS_I915GM(dev))
9191                 dev_priv->display.get_display_clock_speed =
9192                         i915gm_get_display_clock_speed;
9193         else if (IS_I865G(dev))
9194                 dev_priv->display.get_display_clock_speed =
9195                         i865_get_display_clock_speed;
9196         else if (IS_I85X(dev))
9197                 dev_priv->display.get_display_clock_speed =
9198                         i855_get_display_clock_speed;
9199         else /* 852, 830 */
9200                 dev_priv->display.get_display_clock_speed =
9201                         i830_get_display_clock_speed;
9202
9203         /* For FIFO watermark updates */
9204         if (HAS_PCH_SPLIT(dev)) {
9205                 dev_priv->display.force_wake_get = __gen6_gt_force_wake_get;
9206                 dev_priv->display.force_wake_put = __gen6_gt_force_wake_put;
9207
9208                 /* IVB configs may use multi-threaded forcewake */
9209                 if (IS_IVYBRIDGE(dev)) {
9210                         u32     ecobus;
9211
9212                         /* A small trick here - if the bios hasn't configured MT forcewake,
9213                          * and if the device is in RC6, then force_wake_mt_get will not wake
9214                          * the device and the ECOBUS read will return zero. Which will be
9215                          * (correctly) interpreted by the test below as MT forcewake being
9216                          * disabled.
9217                          */
9218                         mutex_lock(&dev->struct_mutex);
9219                         __gen6_gt_force_wake_mt_get(dev_priv);
9220                         ecobus = I915_READ_NOTRACE(ECOBUS);
9221                         __gen6_gt_force_wake_mt_put(dev_priv);
9222                         mutex_unlock(&dev->struct_mutex);
9223
9224                         if (ecobus & FORCEWAKE_MT_ENABLE) {
9225                                 DRM_DEBUG_KMS("Using MT version of forcewake\n");
9226                                 dev_priv->display.force_wake_get =
9227                                         __gen6_gt_force_wake_mt_get;
9228                                 dev_priv->display.force_wake_put =
9229                                         __gen6_gt_force_wake_mt_put;
9230                         }
9231                 }
9232
9233                 if (HAS_PCH_IBX(dev))
9234                         dev_priv->display.init_pch_clock_gating = ibx_init_clock_gating;
9235                 else if (HAS_PCH_CPT(dev))
9236                         dev_priv->display.init_pch_clock_gating = cpt_init_clock_gating;
9237
9238                 if (IS_GEN5(dev)) {
9239                         if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
9240                                 dev_priv->display.update_wm = ironlake_update_wm;
9241                         else {
9242                                 DRM_DEBUG_KMS("Failed to get proper latency. "
9243                                               "Disable CxSR\n");
9244                                 dev_priv->display.update_wm = NULL;
9245                         }
9246                         dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
9247                         dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
9248                         dev_priv->display.write_eld = ironlake_write_eld;
9249                 } else if (IS_GEN6(dev)) {
9250                         if (SNB_READ_WM0_LATENCY()) {
9251                                 dev_priv->display.update_wm = sandybridge_update_wm;
9252                                 dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
9253                         } else {
9254                                 DRM_DEBUG_KMS("Failed to read display plane latency. "
9255                                               "Disable CxSR\n");
9256                                 dev_priv->display.update_wm = NULL;
9257                         }
9258                         dev_priv->display.fdi_link_train = gen6_fdi_link_train;
9259                         dev_priv->display.init_clock_gating = gen6_init_clock_gating;
9260                         dev_priv->display.write_eld = ironlake_write_eld;
9261                 } else if (IS_IVYBRIDGE(dev)) {
9262                         /* FIXME: detect B0+ stepping and use auto training */
9263                         dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
9264                         if (SNB_READ_WM0_LATENCY()) {
9265                                 dev_priv->display.update_wm = sandybridge_update_wm;
9266                                 dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
9267                         } else {
9268                                 DRM_DEBUG_KMS("Failed to read display plane latency. "
9269                                               "Disable CxSR\n");
9270                                 dev_priv->display.update_wm = NULL;
9271                         }
9272                         dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
9273                         dev_priv->display.write_eld = ironlake_write_eld;
9274                 } else
9275                         dev_priv->display.update_wm = NULL;
9276         } else if (IS_VALLEYVIEW(dev)) {
9277                 dev_priv->display.update_wm = valleyview_update_wm;
9278                 dev_priv->display.init_clock_gating =
9279                         valleyview_init_clock_gating;
9280         } else if (IS_PINEVIEW(dev)) {
9281                 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
9282                                             dev_priv->is_ddr3,
9283                                             dev_priv->fsb_freq,
9284                                             dev_priv->mem_freq)) {
9285                         DRM_INFO("failed to find known CxSR latency "
9286                                  "(found ddr%s fsb freq %d, mem freq %d), "
9287                                  "disabling CxSR\n",
9288                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
9289                                  dev_priv->fsb_freq, dev_priv->mem_freq);
9290                         /* Disable CxSR and never update its watermark again */
9291                         pineview_disable_cxsr(dev);
9292                         dev_priv->display.update_wm = NULL;
9293                 } else
9294                         dev_priv->display.update_wm = pineview_update_wm;
9295                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
9296         } else if (IS_G4X(dev)) {
9297                 dev_priv->display.write_eld = g4x_write_eld;
9298                 dev_priv->display.update_wm = g4x_update_wm;
9299                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
9300         } else if (IS_GEN4(dev)) {
9301                 dev_priv->display.update_wm = i965_update_wm;
9302                 if (IS_CRESTLINE(dev))
9303                         dev_priv->display.init_clock_gating = crestline_init_clock_gating;
9304                 else if (IS_BROADWATER(dev))
9305                         dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
9306         } else if (IS_GEN3(dev)) {
9307                 dev_priv->display.update_wm = i9xx_update_wm;
9308                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
9309                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
9310         } else if (IS_I865G(dev)) {
9311                 dev_priv->display.update_wm = i830_update_wm;
9312                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
9313                 dev_priv->display.get_fifo_size = i830_get_fifo_size;
9314         } else if (IS_I85X(dev)) {
9315                 dev_priv->display.update_wm = i9xx_update_wm;
9316                 dev_priv->display.get_fifo_size = i85x_get_fifo_size;
9317                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
9318         } else {
9319                 dev_priv->display.update_wm = i830_update_wm;
9320                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
9321                 if (IS_845G(dev))
9322                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
9323                 else
9324                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
9325         }
9326
9327         /* Default just returns -ENODEV to indicate unsupported */
9328         dev_priv->display.queue_flip = intel_default_queue_flip;
9329
9330         switch (INTEL_INFO(dev)->gen) {
9331         case 2:
9332                 dev_priv->display.queue_flip = intel_gen2_queue_flip;
9333                 break;
9334
9335         case 3:
9336                 dev_priv->display.queue_flip = intel_gen3_queue_flip;
9337                 break;
9338
9339         case 4:
9340         case 5:
9341                 dev_priv->display.queue_flip = intel_gen4_queue_flip;
9342                 break;
9343
9344         case 6:
9345                 dev_priv->display.queue_flip = intel_gen6_queue_flip;
9346                 break;
9347         case 7:
9348                 dev_priv->display.queue_flip = intel_gen7_queue_flip;
9349                 break;
9350         }
9351 }
9352
9353 /*
9354  * Some BIOSes insist on assuming the GPU's pipe A is enabled at suspend,
9355  * resume, or other times.  This quirk makes sure that's the case for
9356  * affected systems.
9357  */
9358 static void quirk_pipea_force(struct drm_device *dev)
9359 {
9360         struct drm_i915_private *dev_priv = dev->dev_private;
9361
9362         dev_priv->quirks |= QUIRK_PIPEA_FORCE;
9363         DRM_DEBUG_DRIVER("applying pipe a force quirk\n");
9364 }
9365
9366 /*
9367  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
9368  */
9369 static void quirk_ssc_force_disable(struct drm_device *dev)
9370 {
9371         struct drm_i915_private *dev_priv = dev->dev_private;
9372         dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
9373 }
9374
9375 /*
9376  * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight
9377  * brightness value
9378  */
9379 static void quirk_invert_brightness(struct drm_device *dev)
9380 {
9381         struct drm_i915_private *dev_priv = dev->dev_private;
9382         dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS;
9383 }
9384
9385 struct intel_quirk {
9386         int device;
9387         int subsystem_vendor;
9388         int subsystem_device;
9389         void (*hook)(struct drm_device *dev);
9390 };
9391
9392 struct intel_quirk intel_quirks[] = {
9393         /* HP Mini needs pipe A force quirk (LP: #322104) */
9394         { 0x27ae, 0x103c, 0x361a, quirk_pipea_force },
9395
9396         /* Thinkpad R31 needs pipe A force quirk */
9397         { 0x3577, 0x1014, 0x0505, quirk_pipea_force },
9398         /* Toshiba Protege R-205, S-209 needs pipe A force quirk */
9399         { 0x2592, 0x1179, 0x0001, quirk_pipea_force },
9400
9401         /* ThinkPad X30 needs pipe A force quirk (LP: #304614) */
9402         { 0x3577,  0x1014, 0x0513, quirk_pipea_force },
9403         /* ThinkPad X40 needs pipe A force quirk */
9404
9405         /* ThinkPad T60 needs pipe A force quirk (bug #16494) */
9406         { 0x2782, 0x17aa, 0x201a, quirk_pipea_force },
9407
9408         /* 855 & before need to leave pipe A & dpll A up */
9409         { 0x3582, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
9410         { 0x2562, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
9411
9412         /* Lenovo U160 cannot use SSC on LVDS */
9413         { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
9414
9415         /* Sony Vaio Y cannot use SSC on LVDS */
9416         { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable },
9417
9418         /* Acer Aspire 5734Z must invert backlight brightness */
9419         { 0x2a42, 0x1025, 0x0459, quirk_invert_brightness },
9420 };
9421
9422 static void intel_init_quirks(struct drm_device *dev)
9423 {
9424         struct pci_dev *d = dev->pdev;
9425         int i;
9426
9427         for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) {
9428                 struct intel_quirk *q = &intel_quirks[i];
9429
9430                 if (d->device == q->device &&
9431                     (d->subsystem_vendor == q->subsystem_vendor ||
9432                      q->subsystem_vendor == PCI_ANY_ID) &&
9433                     (d->subsystem_device == q->subsystem_device ||
9434                      q->subsystem_device == PCI_ANY_ID))
9435                         q->hook(dev);
9436         }
9437 }
9438
9439 /* Disable the VGA plane that we never use */
9440 static void i915_disable_vga(struct drm_device *dev)
9441 {
9442         struct drm_i915_private *dev_priv = dev->dev_private;
9443         u8 sr1;
9444         u32 vga_reg;
9445
9446         if (HAS_PCH_SPLIT(dev))
9447                 vga_reg = CPU_VGACNTRL;
9448         else
9449                 vga_reg = VGACNTRL;
9450
9451         vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
9452         outb(1, VGA_SR_INDEX);
9453         sr1 = inb(VGA_SR_DATA);
9454         outb(sr1 | 1<<5, VGA_SR_DATA);
9455         vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
9456         udelay(300);
9457
9458         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
9459         POSTING_READ(vga_reg);
9460 }
9461
9462 void intel_modeset_init(struct drm_device *dev)
9463 {
9464         struct drm_i915_private *dev_priv = dev->dev_private;
9465         int i, ret;
9466
9467         drm_mode_config_init(dev);
9468
9469         dev->mode_config.min_width = 0;
9470         dev->mode_config.min_height = 0;
9471
9472         dev->mode_config.preferred_depth = 24;
9473         dev->mode_config.prefer_shadow = 1;
9474
9475         dev->mode_config.funcs = (void *)&intel_mode_funcs;
9476
9477         intel_init_quirks(dev);
9478
9479         intel_init_display(dev);
9480
9481         if (IS_GEN2(dev)) {
9482                 dev->mode_config.max_width = 2048;
9483                 dev->mode_config.max_height = 2048;
9484         } else if (IS_GEN3(dev)) {
9485                 dev->mode_config.max_width = 4096;
9486                 dev->mode_config.max_height = 4096;
9487         } else {
9488                 dev->mode_config.max_width = 8192;
9489                 dev->mode_config.max_height = 8192;
9490         }
9491         dev->mode_config.fb_base = dev->agp->base;
9492
9493         DRM_DEBUG_KMS("%d display pipe%s available.\n",
9494                       dev_priv->num_pipe, dev_priv->num_pipe > 1 ? "s" : "");
9495
9496         for (i = 0; i < dev_priv->num_pipe; i++) {
9497                 intel_crtc_init(dev, i);
9498                 ret = intel_plane_init(dev, i);
9499                 if (ret)
9500                         DRM_DEBUG_KMS("plane %d init failed: %d\n", i, ret);
9501         }
9502
9503         /* Just disable it once at startup */
9504         i915_disable_vga(dev);
9505         intel_setup_outputs(dev);
9506
9507         intel_init_clock_gating(dev);
9508
9509         if (IS_IRONLAKE_M(dev)) {
9510                 ironlake_enable_drps(dev);
9511                 intel_init_emon(dev);
9512         }
9513
9514         if (IS_GEN6(dev) || IS_GEN7(dev)) {
9515                 gen6_enable_rps(dev_priv);
9516                 gen6_update_ring_freq(dev_priv);
9517         }
9518
9519         INIT_WORK(&dev_priv->idle_work, intel_idle_update);
9520         setup_timer(&dev_priv->idle_timer, intel_gpu_idle_timer,
9521                     (unsigned long)dev);
9522 }
9523
9524 void intel_modeset_gem_init(struct drm_device *dev)
9525 {
9526         if (IS_IRONLAKE_M(dev))
9527                 ironlake_enable_rc6(dev);
9528
9529         intel_setup_overlay(dev);
9530 }
9531
9532 void intel_modeset_cleanup(struct drm_device *dev)
9533 {
9534         struct drm_i915_private *dev_priv = dev->dev_private;
9535         struct drm_crtc *crtc;
9536         struct intel_crtc *intel_crtc;
9537
9538         drm_kms_helper_poll_fini(dev);
9539         mutex_lock(&dev->struct_mutex);
9540
9541         intel_unregister_dsm_handler();
9542
9543
9544         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
9545                 /* Skip inactive CRTCs */
9546                 if (!crtc->fb)
9547                         continue;
9548
9549                 intel_crtc = to_intel_crtc(crtc);
9550                 intel_increase_pllclock(crtc);
9551         }
9552
9553         intel_disable_fbc(dev);
9554
9555         if (IS_IRONLAKE_M(dev))
9556                 ironlake_disable_drps(dev);
9557         if (IS_GEN6(dev) || IS_GEN7(dev))
9558                 gen6_disable_rps(dev);
9559
9560         if (IS_IRONLAKE_M(dev))
9561                 ironlake_disable_rc6(dev);
9562
9563         if (IS_VALLEYVIEW(dev))
9564                 vlv_init_dpio(dev);
9565
9566         mutex_unlock(&dev->struct_mutex);
9567
9568         /* Disable the irq before mode object teardown, for the irq might
9569          * enqueue unpin/hotplug work. */
9570         drm_irq_uninstall(dev);
9571         cancel_work_sync(&dev_priv->hotplug_work);
9572         cancel_work_sync(&dev_priv->rps_work);
9573
9574         /* flush any delayed tasks or pending work */
9575         flush_scheduled_work();
9576
9577         /* Shut off idle work before the crtcs get freed. */
9578         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
9579                 intel_crtc = to_intel_crtc(crtc);
9580                 del_timer_sync(&intel_crtc->idle_timer);
9581         }
9582         del_timer_sync(&dev_priv->idle_timer);
9583         cancel_work_sync(&dev_priv->idle_work);
9584
9585         drm_mode_config_cleanup(dev);
9586 }
9587
9588 /*
9589  * Return which encoder is currently attached for connector.
9590  */
9591 struct drm_encoder *intel_best_encoder(struct drm_connector *connector)
9592 {
9593         return &intel_attached_encoder(connector)->base;
9594 }
9595
9596 void intel_connector_attach_encoder(struct intel_connector *connector,
9597                                     struct intel_encoder *encoder)
9598 {
9599         connector->encoder = encoder;
9600         drm_mode_connector_attach_encoder(&connector->base,
9601                                           &encoder->base);
9602 }
9603
9604 /*
9605  * set vga decode state - true == enable VGA decode
9606  */
9607 int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
9608 {
9609         struct drm_i915_private *dev_priv = dev->dev_private;
9610         u16 gmch_ctrl;
9611
9612         pci_read_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, &gmch_ctrl);
9613         if (state)
9614                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
9615         else
9616                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
9617         pci_write_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, gmch_ctrl);
9618         return 0;
9619 }
9620
9621 #ifdef CONFIG_DEBUG_FS
9622 #include <linux/seq_file.h>
9623
9624 struct intel_display_error_state {
9625         struct intel_cursor_error_state {
9626                 u32 control;
9627                 u32 position;
9628                 u32 base;
9629                 u32 size;
9630         } cursor[2];
9631
9632         struct intel_pipe_error_state {
9633                 u32 conf;
9634                 u32 source;
9635
9636                 u32 htotal;
9637                 u32 hblank;
9638                 u32 hsync;
9639                 u32 vtotal;
9640                 u32 vblank;
9641                 u32 vsync;
9642         } pipe[2];
9643
9644         struct intel_plane_error_state {
9645                 u32 control;
9646                 u32 stride;
9647                 u32 size;
9648                 u32 pos;
9649                 u32 addr;
9650                 u32 surface;
9651                 u32 tile_offset;
9652         } plane[2];
9653 };
9654
9655 struct intel_display_error_state *
9656 intel_display_capture_error_state(struct drm_device *dev)
9657 {
9658         drm_i915_private_t *dev_priv = dev->dev_private;
9659         struct intel_display_error_state *error;
9660         int i;
9661
9662         error = kmalloc(sizeof(*error), GFP_ATOMIC);
9663         if (error == NULL)
9664                 return NULL;
9665
9666         for (i = 0; i < 2; i++) {
9667                 error->cursor[i].control = I915_READ(CURCNTR(i));
9668                 error->cursor[i].position = I915_READ(CURPOS(i));
9669                 error->cursor[i].base = I915_READ(CURBASE(i));
9670
9671                 error->plane[i].control = I915_READ(DSPCNTR(i));
9672                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
9673                 error->plane[i].size = I915_READ(DSPSIZE(i));
9674                 error->plane[i].pos = I915_READ(DSPPOS(i));
9675                 error->plane[i].addr = I915_READ(DSPADDR(i));
9676                 if (INTEL_INFO(dev)->gen >= 4) {
9677                         error->plane[i].surface = I915_READ(DSPSURF(i));
9678                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
9679                 }
9680
9681                 error->pipe[i].conf = I915_READ(PIPECONF(i));
9682                 error->pipe[i].source = I915_READ(PIPESRC(i));
9683                 error->pipe[i].htotal = I915_READ(HTOTAL(i));
9684                 error->pipe[i].hblank = I915_READ(HBLANK(i));
9685                 error->pipe[i].hsync = I915_READ(HSYNC(i));
9686                 error->pipe[i].vtotal = I915_READ(VTOTAL(i));
9687                 error->pipe[i].vblank = I915_READ(VBLANK(i));
9688                 error->pipe[i].vsync = I915_READ(VSYNC(i));
9689         }
9690
9691         return error;
9692 }
9693
9694 void
9695 intel_display_print_error_state(struct seq_file *m,
9696                                 struct drm_device *dev,
9697                                 struct intel_display_error_state *error)
9698 {
9699         int i;
9700
9701         for (i = 0; i < 2; i++) {
9702                 seq_printf(m, "Pipe [%d]:\n", i);
9703                 seq_printf(m, "  CONF: %08x\n", error->pipe[i].conf);
9704                 seq_printf(m, "  SRC: %08x\n", error->pipe[i].source);
9705                 seq_printf(m, "  HTOTAL: %08x\n", error->pipe[i].htotal);
9706                 seq_printf(m, "  HBLANK: %08x\n", error->pipe[i].hblank);
9707                 seq_printf(m, "  HSYNC: %08x\n", error->pipe[i].hsync);
9708                 seq_printf(m, "  VTOTAL: %08x\n", error->pipe[i].vtotal);
9709                 seq_printf(m, "  VBLANK: %08x\n", error->pipe[i].vblank);
9710                 seq_printf(m, "  VSYNC: %08x\n", error->pipe[i].vsync);
9711
9712                 seq_printf(m, "Plane [%d]:\n", i);
9713                 seq_printf(m, "  CNTR: %08x\n", error->plane[i].control);
9714                 seq_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
9715                 seq_printf(m, "  SIZE: %08x\n", error->plane[i].size);
9716                 seq_printf(m, "  POS: %08x\n", error->plane[i].pos);
9717                 seq_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
9718                 if (INTEL_INFO(dev)->gen >= 4) {
9719                         seq_printf(m, "  SURF: %08x\n", error->plane[i].surface);
9720                         seq_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
9721                 }
9722
9723                 seq_printf(m, "Cursor [%d]:\n", i);
9724                 seq_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
9725                 seq_printf(m, "  POS: %08x\n", error->cursor[i].position);
9726                 seq_printf(m, "  BASE: %08x\n", error->cursor[i].base);
9727         }
9728 }
9729 #endif