drivers/gpu/drm/i915/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/dmi.h>
  28 #include <linux/module.h>
  29 #include <linux/input.h>
  30 #include <linux/i2c.h>
  31 #include <linux/kernel.h>
  32 #include <linux/slab.h>
  33 #include <linux/vgaarb.h>
  34 #include <drm/drm_edid.h>
  35 #include <drm/drmP.h>
  36 #include "intel_drv.h"
  37 #include <drm/i915_drm.h>
  38 #include "i915_drv.h"
  39 #include "i915_trace.h"
  40 #include <drm/drm_atomic.h>
  41 #include <drm/drm_atomic_helper.h>
  42 #include <drm/drm_dp_helper.h>
  43 #include <drm/drm_crtc_helper.h>
  44 #include <drm/drm_plane_helper.h>
  45 #include <drm/drm_rect.h>
  46 #include <linux/dma_remapping.h>
  47
  48 /* Primary plane formats for gen <= 3 */
  49 static const uint32_t i8xx_primary_formats[] = {
  50         DRM_FORMAT_C8,
  51         DRM_FORMAT_RGB565,
  52         DRM_FORMAT_XRGB1555,
  53         DRM_FORMAT_XRGB8888,
  54 };
  55
  56 /* Primary plane formats for gen >= 4 */
  57 static const uint32_t i965_primary_formats[] = {
  58         DRM_FORMAT_C8,
  59         DRM_FORMAT_RGB565,
  60         DRM_FORMAT_XRGB8888,
  61         DRM_FORMAT_XBGR8888,
  62         DRM_FORMAT_XRGB2101010,
  63         DRM_FORMAT_XBGR2101010,
  64 };
  65
  66 static const uint32_t skl_primary_formats[] = {
  67         DRM_FORMAT_C8,
  68         DRM_FORMAT_RGB565,
  69         DRM_FORMAT_XRGB8888,
  70         DRM_FORMAT_XBGR8888,
  71         DRM_FORMAT_ARGB8888,
  72         DRM_FORMAT_ABGR8888,
  73         DRM_FORMAT_XRGB2101010,
  74         DRM_FORMAT_XBGR2101010,
  75         DRM_FORMAT_YUYV,
  76         DRM_FORMAT_YVYU,
  77         DRM_FORMAT_UYVY,
  78         DRM_FORMAT_VYUY,
  79 };
  80
  81 /* Cursor formats */
  82 static const uint32_t intel_cursor_formats[] = {
  83         DRM_FORMAT_ARGB8888,
  84 };
  85
  86 static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on);
  87
  88 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
  89                                 struct intel_crtc_state *pipe_config);
  90 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
  91                                    struct intel_crtc_state *pipe_config);
  92
  93 static int intel_framebuffer_init(struct drm_device *dev,
  94                                   struct intel_framebuffer *ifb,
  95                                   struct drm_mode_fb_cmd2 *mode_cmd,
  96                                   struct drm_i915_gem_object *obj);
  97 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc);
  98 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc);
  99 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
 100                                          struct intel_link_m_n *m_n,
 101                                          struct intel_link_m_n *m2_n2);
 102 static void ironlake_set_pipeconf(struct drm_crtc *crtc);
 103 static void haswell_set_pipeconf(struct drm_crtc *crtc);
 104 static void intel_set_pipe_csc(struct drm_crtc *crtc);
 105 static void vlv_prepare_pll(struct intel_crtc *crtc,
 106                             const struct intel_crtc_state *pipe_config);
 107 static void chv_prepare_pll(struct intel_crtc *crtc,
 108                             const struct intel_crtc_state *pipe_config);
 109 static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 110 static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 111 static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc,
 112         struct intel_crtc_state *crtc_state);
 113 static int i9xx_get_refclk(const struct intel_crtc_state *crtc_state,
 114                            int num_connectors);
 115 static void skylake_pfit_enable(struct intel_crtc *crtc);
 116 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force);
 117 static void ironlake_pfit_enable(struct intel_crtc *crtc);
 118 static void intel_modeset_setup_hw_state(struct drm_device *dev);
 119 static void intel_pre_disable_primary(struct drm_crtc *crtc);
 120
 121 typedef struct {
 122         int     min, max;
 123 } intel_range_t;
 124
 125 typedef struct {
 126         int     dot_limit;
 127         int     p2_slow, p2_fast;
 128 } intel_p2_t;
 129
 130 typedef struct intel_limit intel_limit_t;
 131 struct intel_limit {
 132         intel_range_t   dot, vco, n, m, m1, m2, p, p1;
 133         intel_p2_t          p2;
 134 };
 135
 136 /* returns HPLL frequency in kHz */
 137 static int valleyview_get_vco(struct drm_i915_private *dev_priv)
 138 {
 139         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 140
 141         /* Obtain SKU information */
 142         mutex_lock(&dev_priv->sb_lock);
 143         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 144                 CCK_FUSE_HPLL_FREQ_MASK;
 145         mutex_unlock(&dev_priv->sb_lock);
 146
 147         return vco_freq[hpll_freq] * 1000;
 148 }
 149
 150 static int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 151                                   const char *name, u32 reg)
 152 {
 153         u32 val;
 154         int divider;
 155
 156         if (dev_priv->hpll_freq == 0)
 157                 dev_priv->hpll_freq = valleyview_get_vco(dev_priv);
 158
 159         mutex_lock(&dev_priv->sb_lock);
 160         val = vlv_cck_read(dev_priv, reg);
 161         mutex_unlock(&dev_priv->sb_lock);
 162
 163         divider = val & CCK_FREQUENCY_VALUES;
 164
 165         WARN((val & CCK_FREQUENCY_STATUS) !=
 166              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 167              "%s change in progress\n", name);
 168
 169         return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, divider + 1);
 170 }
 171
 172 int
 173 intel_pch_rawclk(struct drm_device *dev)
 174 {
 175         struct drm_i915_private *dev_priv = dev->dev_private;
 176
 177         WARN_ON(!HAS_PCH_SPLIT(dev));
 178
 179         return I915_READ(PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK;
 180 }
 181
 182 /* hrawclock is 1/4 the FSB frequency */
 183 int intel_hrawclk(struct drm_device *dev)
 184 {
 185         struct drm_i915_private *dev_priv = dev->dev_private;
 186         uint32_t clkcfg;
 187
 188         /* There is no CLKCFG reg in Valleyview. VLV hrawclk is 200 MHz */
 189         if (IS_VALLEYVIEW(dev))
 190                 return 200;
 191
 192         clkcfg = I915_READ(CLKCFG);
 193         switch (clkcfg & CLKCFG_FSB_MASK) {
 194         case CLKCFG_FSB_400:
 195                 return 100;
 196         case CLKCFG_FSB_533:
 197                 return 133;
 198         case CLKCFG_FSB_667:
 199                 return 166;
 200         case CLKCFG_FSB_800:
 201                 return 200;
 202         case CLKCFG_FSB_1067:
 203                 return 266;
 204         case CLKCFG_FSB_1333:
 205                 return 333;
 206         /* these two are just a guess; one of them might be right */
 207         case CLKCFG_FSB_1600:
 208         case CLKCFG_FSB_1600_ALT:
 209                 return 400;
 210         default:
 211                 return 133;
 212         }
 213 }
 214
 215 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 216 {
 217         if (!IS_VALLEYVIEW(dev_priv))
 218                 return;
 219
 220         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 221                                                       CCK_CZ_CLOCK_CONTROL);
 222
 223         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 224 }
 225
 226 static inline u32 /* units of 100MHz */
 227 intel_fdi_link_freq(struct drm_device *dev)
 228 {
 229         if (IS_GEN5(dev)) {
 230                 struct drm_i915_private *dev_priv = dev->dev_private;
 231                 return (I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK) + 2;
 232         } else
 233                 return 27;
 234 }
 235
 236 static const intel_limit_t intel_limits_i8xx_dac = {
 237         .dot = { .min = 25000, .max = 350000 },
 238         .vco = { .min = 908000, .max = 1512000 },
 239         .n = { .min = 2, .max = 16 },
 240         .m = { .min = 96, .max = 140 },
 241         .m1 = { .min = 18, .max = 26 },
 242         .m2 = { .min = 6, .max = 16 },
 243         .p = { .min = 4, .max = 128 },
 244         .p1 = { .min = 2, .max = 33 },
 245         .p2 = { .dot_limit = 165000,
 246                 .p2_slow = 4, .p2_fast = 2 },
 247 };
 248
 249 static const intel_limit_t intel_limits_i8xx_dvo = {
 250         .dot = { .min = 25000, .max = 350000 },
 251         .vco = { .min = 908000, .max = 1512000 },
 252         .n = { .min = 2, .max = 16 },
 253         .m = { .min = 96, .max = 140 },
 254         .m1 = { .min = 18, .max = 26 },
 255         .m2 = { .min = 6, .max = 16 },
 256         .p = { .min = 4, .max = 128 },
 257         .p1 = { .min = 2, .max = 33 },
 258         .p2 = { .dot_limit = 165000,
 259                 .p2_slow = 4, .p2_fast = 4 },
 260 };
 261
 262 static const intel_limit_t intel_limits_i8xx_lvds = {
 263         .dot = { .min = 25000, .max = 350000 },
 264         .vco = { .min = 908000, .max = 1512000 },
 265         .n = { .min = 2, .max = 16 },
 266         .m = { .min = 96, .max = 140 },
 267         .m1 = { .min = 18, .max = 26 },
 268         .m2 = { .min = 6, .max = 16 },
 269         .p = { .min = 4, .max = 128 },
 270         .p1 = { .min = 1, .max = 6 },
 271         .p2 = { .dot_limit = 165000,
 272                 .p2_slow = 14, .p2_fast = 7 },
 273 };
 274
 275 static const intel_limit_t intel_limits_i9xx_sdvo = {
 276         .dot = { .min = 20000, .max = 400000 },
 277         .vco = { .min = 1400000, .max = 2800000 },
 278         .n = { .min = 1, .max = 6 },
 279         .m = { .min = 70, .max = 120 },
 280         .m1 = { .min = 8, .max = 18 },
 281         .m2 = { .min = 3, .max = 7 },
 282         .p = { .min = 5, .max = 80 },
 283         .p1 = { .min = 1, .max = 8 },
 284         .p2 = { .dot_limit = 200000,
 285                 .p2_slow = 10, .p2_fast = 5 },
 286 };
 287
 288 static const intel_limit_t intel_limits_i9xx_lvds = {
 289         .dot = { .min = 20000, .max = 400000 },
 290         .vco = { .min = 1400000, .max = 2800000 },
 291         .n = { .min = 1, .max = 6 },
 292         .m = { .min = 70, .max = 120 },
 293         .m1 = { .min = 8, .max = 18 },
 294         .m2 = { .min = 3, .max = 7 },
 295         .p = { .min = 7, .max = 98 },
 296         .p1 = { .min = 1, .max = 8 },
 297         .p2 = { .dot_limit = 112000,
 298                 .p2_slow = 14, .p2_fast = 7 },
 299 };
 300
 301
 302 static const intel_limit_t intel_limits_g4x_sdvo = {
 303         .dot = { .min = 25000, .max = 270000 },
 304         .vco = { .min = 1750000, .max = 3500000},
 305         .n = { .min = 1, .max = 4 },
 306         .m = { .min = 104, .max = 138 },
 307         .m1 = { .min = 17, .max = 23 },
 308         .m2 = { .min = 5, .max = 11 },
 309         .p = { .min = 10, .max = 30 },
 310         .p1 = { .min = 1, .max = 3},
 311         .p2 = { .dot_limit = 270000,
 312                 .p2_slow = 10,
 313                 .p2_fast = 10
 314         },
 315 };
 316
 317 static const intel_limit_t intel_limits_g4x_hdmi = {
 318         .dot = { .min = 22000, .max = 400000 },
 319         .vco = { .min = 1750000, .max = 3500000},
 320         .n = { .min = 1, .max = 4 },
 321         .m = { .min = 104, .max = 138 },
 322         .m1 = { .min = 16, .max = 23 },
 323         .m2 = { .min = 5, .max = 11 },
 324         .p = { .min = 5, .max = 80 },
 325         .p1 = { .min = 1, .max = 8},
 326         .p2 = { .dot_limit = 165000,
 327                 .p2_slow = 10, .p2_fast = 5 },
 328 };
 329
 330 static const intel_limit_t intel_limits_g4x_single_channel_lvds = {
 331         .dot = { .min = 20000, .max = 115000 },
 332         .vco = { .min = 1750000, .max = 3500000 },
 333         .n = { .min = 1, .max = 3 },
 334         .m = { .min = 104, .max = 138 },
 335         .m1 = { .min = 17, .max = 23 },
 336         .m2 = { .min = 5, .max = 11 },
 337         .p = { .min = 28, .max = 112 },
 338         .p1 = { .min = 2, .max = 8 },
 339         .p2 = { .dot_limit = 0,
 340                 .p2_slow = 14, .p2_fast = 14
 341         },
 342 };
 343
 344 static const intel_limit_t intel_limits_g4x_dual_channel_lvds = {
 345         .dot = { .min = 80000, .max = 224000 },
 346         .vco = { .min = 1750000, .max = 3500000 },
 347         .n = { .min = 1, .max = 3 },
 348         .m = { .min = 104, .max = 138 },
 349         .m1 = { .min = 17, .max = 23 },
 350         .m2 = { .min = 5, .max = 11 },
 351         .p = { .min = 14, .max = 42 },
 352         .p1 = { .min = 2, .max = 6 },
 353         .p2 = { .dot_limit = 0,
 354                 .p2_slow = 7, .p2_fast = 7
 355         },
 356 };
 357
 358 static const intel_limit_t intel_limits_pineview_sdvo = {
 359         .dot = { .min = 20000, .max = 400000},
 360         .vco = { .min = 1700000, .max = 3500000 },
 361         /* Pineview's Ncounter is a ring counter */
 362         .n = { .min = 3, .max = 6 },
 363         .m = { .min = 2, .max = 256 },
 364         /* Pineview only has one combined m divider, which we treat as m2. */
 365         .m1 = { .min = 0, .max = 0 },
 366         .m2 = { .min = 0, .max = 254 },
 367         .p = { .min = 5, .max = 80 },
 368         .p1 = { .min = 1, .max = 8 },
 369         .p2 = { .dot_limit = 200000,
 370                 .p2_slow = 10, .p2_fast = 5 },
 371 };
 372
 373 static const intel_limit_t intel_limits_pineview_lvds = {
 374         .dot = { .min = 20000, .max = 400000 },
 375         .vco = { .min = 1700000, .max = 3500000 },
 376         .n = { .min = 3, .max = 6 },
 377         .m = { .min = 2, .max = 256 },
 378         .m1 = { .min = 0, .max = 0 },
 379         .m2 = { .min = 0, .max = 254 },
 380         .p = { .min = 7, .max = 112 },
 381         .p1 = { .min = 1, .max = 8 },
 382         .p2 = { .dot_limit = 112000,
 383                 .p2_slow = 14, .p2_fast = 14 },
 384 };
 385
 386 /* Ironlake / Sandybridge
 387  *
 388  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 389  * the range value for them is (actual_value - 2).
 390  */
 391 static const intel_limit_t intel_limits_ironlake_dac = {
 392         .dot = { .min = 25000, .max = 350000 },
 393         .vco = { .min = 1760000, .max = 3510000 },
 394         .n = { .min = 1, .max = 5 },
 395         .m = { .min = 79, .max = 127 },
 396         .m1 = { .min = 12, .max = 22 },
 397         .m2 = { .min = 5, .max = 9 },
 398         .p = { .min = 5, .max = 80 },
 399         .p1 = { .min = 1, .max = 8 },
 400         .p2 = { .dot_limit = 225000,
 401                 .p2_slow = 10, .p2_fast = 5 },
 402 };
 403
 404 static const intel_limit_t intel_limits_ironlake_single_lvds = {
 405         .dot = { .min = 25000, .max = 350000 },
 406         .vco = { .min = 1760000, .max = 3510000 },
 407         .n = { .min = 1, .max = 3 },
 408         .m = { .min = 79, .max = 118 },
 409         .m1 = { .min = 12, .max = 22 },
 410         .m2 = { .min = 5, .max = 9 },
 411         .p = { .min = 28, .max = 112 },
 412         .p1 = { .min = 2, .max = 8 },
 413         .p2 = { .dot_limit = 225000,
 414                 .p2_slow = 14, .p2_fast = 14 },
 415 };
 416
 417 static const intel_limit_t intel_limits_ironlake_dual_lvds = {
 418         .dot = { .min = 25000, .max = 350000 },
 419         .vco = { .min = 1760000, .max = 3510000 },
 420         .n = { .min = 1, .max = 3 },
 421         .m = { .min = 79, .max = 127 },
 422         .m1 = { .min = 12, .max = 22 },
 423         .m2 = { .min = 5, .max = 9 },
 424         .p = { .min = 14, .max = 56 },
 425         .p1 = { .min = 2, .max = 8 },
 426         .p2 = { .dot_limit = 225000,
 427                 .p2_slow = 7, .p2_fast = 7 },
 428 };
 429
 430 /* LVDS 100mhz refclk limits. */
 431 static const intel_limit_t intel_limits_ironlake_single_lvds_100m = {
 432         .dot = { .min = 25000, .max = 350000 },
 433         .vco = { .min = 1760000, .max = 3510000 },
 434         .n = { .min = 1, .max = 2 },
 435         .m = { .min = 79, .max = 126 },
 436         .m1 = { .min = 12, .max = 22 },
 437         .m2 = { .min = 5, .max = 9 },
 438         .p = { .min = 28, .max = 112 },
 439         .p1 = { .min = 2, .max = 8 },
 440         .p2 = { .dot_limit = 225000,
 441                 .p2_slow = 14, .p2_fast = 14 },
 442 };
 443
 444 static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = {
 445         .dot = { .min = 25000, .max = 350000 },
 446         .vco = { .min = 1760000, .max = 3510000 },
 447         .n = { .min = 1, .max = 3 },
 448         .m = { .min = 79, .max = 126 },
 449         .m1 = { .min = 12, .max = 22 },
 450         .m2 = { .min = 5, .max = 9 },
 451         .p = { .min = 14, .max = 42 },
 452         .p1 = { .min = 2, .max = 6 },
 453         .p2 = { .dot_limit = 225000,
 454                 .p2_slow = 7, .p2_fast = 7 },
 455 };
 456
 457 static const intel_limit_t intel_limits_vlv = {
 458          /*
 459           * These are the data rate limits (measured in fast clocks)
 460           * since those are the strictest limits we have. The fast
 461           * clock and actual rate limits are more relaxed, so checking
 462           * them would make no difference.
 463           */
 464         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 465         .vco = { .min = 4000000, .max = 6000000 },
 466         .n = { .min = 1, .max = 7 },
 467         .m1 = { .min = 2, .max = 3 },
 468         .m2 = { .min = 11, .max = 156 },
 469         .p1 = { .min = 2, .max = 3 },
 470         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 471 };
 472
 473 static const intel_limit_t intel_limits_chv = {
 474         /*
 475          * These are the data rate limits (measured in fast clocks)
 476          * since those are the strictest limits we have.  The fast
 477          * clock and actual rate limits are more relaxed, so checking
 478          * them would make no difference.
 479          */
 480         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 481         .vco = { .min = 4800000, .max = 6480000 },
 482         .n = { .min = 1, .max = 1 },
 483         .m1 = { .min = 2, .max = 2 },
 484         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 485         .p1 = { .min = 2, .max = 4 },
 486         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 487 };
 488
 489 static const intel_limit_t intel_limits_bxt = {
 490         /* FIXME: find real dot limits */
 491         .dot = { .min = 0, .max = INT_MAX },
 492         .vco = { .min = 4800000, .max = 6700000 },
 493         .n = { .min = 1, .max = 1 },
 494         .m1 = { .min = 2, .max = 2 },
 495         /* FIXME: find real m2 limits */
 496         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 497         .p1 = { .min = 2, .max = 4 },
 498         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 499 };
 500
 501 static bool
 502 needs_modeset(struct drm_crtc_state *state)
 503 {
 504         return drm_atomic_crtc_needs_modeset(state);
 505 }
 506
 507 /**
 508  * Returns whether any output on the specified pipe is of the specified type
 509  */
 510 bool intel_pipe_has_type(struct intel_crtc *crtc, enum intel_output_type type)
 511 {
 512         struct drm_device *dev = crtc->base.dev;
 513         struct intel_encoder *encoder;
 514
 515         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
 516                 if (encoder->type == type)
 517                         return true;
 518
 519         return false;
 520 }
 521
 522 /**
 523  * Returns whether any output on the specified pipe will have the specified
 524  * type after a staged modeset is complete, i.e., the same as
 525  * intel_pipe_has_type() but looking at encoder->new_crtc instead of
 526  * encoder->crtc.
 527  */
 528 static bool intel_pipe_will_have_type(const struct intel_crtc_state *crtc_state,
 529                                       int type)
 530 {
 531         struct drm_atomic_state *state = crtc_state->base.state;
 532         struct drm_connector *connector;
 533         struct drm_connector_state *connector_state;
 534         struct intel_encoder *encoder;
 535         int i, num_connectors = 0;
 536
 537         for_each_connector_in_state(state, connector, connector_state, i) {
 538                 if (connector_state->crtc != crtc_state->base.crtc)
 539                         continue;
 540
 541                 num_connectors++;
 542
 543                 encoder = to_intel_encoder(connector_state->best_encoder);
 544                 if (encoder->type == type)
 545                         return true;
 546         }
 547
 548         WARN_ON(num_connectors == 0);
 549
 550         return false;
 551 }
 552
 553 static const intel_limit_t *
 554 intel_ironlake_limit(struct intel_crtc_state *crtc_state, int refclk)
 555 {
 556         struct drm_device *dev = crtc_state->base.crtc->dev;
 557         const intel_limit_t *limit;
 558
 559         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 560                 if (intel_is_dual_link_lvds(dev)) {
 561                         if (refclk == 100000)
 562                                 limit = &intel_limits_ironlake_dual_lvds_100m;
 563                         else
 564                                 limit = &intel_limits_ironlake_dual_lvds;
 565                 } else {
 566                         if (refclk == 100000)
 567                                 limit = &intel_limits_ironlake_single_lvds_100m;
 568                         else
 569                                 limit = &intel_limits_ironlake_single_lvds;
 570                 }
 571         } else
 572                 limit = &intel_limits_ironlake_dac;
 573
 574         return limit;
 575 }
 576
 577 static const intel_limit_t *
 578 intel_g4x_limit(struct intel_crtc_state *crtc_state)
 579 {
 580         struct drm_device *dev = crtc_state->base.crtc->dev;
 581         const intel_limit_t *limit;
 582
 583         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 584                 if (intel_is_dual_link_lvds(dev))
 585                         limit = &intel_limits_g4x_dual_channel_lvds;
 586                 else
 587                         limit = &intel_limits_g4x_single_channel_lvds;
 588         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI) ||
 589                    intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
 590                 limit = &intel_limits_g4x_hdmi;
 591         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO)) {
 592                 limit = &intel_limits_g4x_sdvo;
 593         } else /* The option is for other outputs */
 594                 limit = &intel_limits_i9xx_sdvo;
 595
 596         return limit;
 597 }
 598
 599 static const intel_limit_t *
 600 intel_limit(struct intel_crtc_state *crtc_state, int refclk)
 601 {
 602         struct drm_device *dev = crtc_state->base.crtc->dev;
 603         const intel_limit_t *limit;
 604
 605         if (IS_BROXTON(dev))
 606                 limit = &intel_limits_bxt;
 607         else if (HAS_PCH_SPLIT(dev))
 608                 limit = intel_ironlake_limit(crtc_state, refclk);
 609         else if (IS_G4X(dev)) {
 610                 limit = intel_g4x_limit(crtc_state);
 611         } else if (IS_PINEVIEW(dev)) {
 612                 if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
 613                         limit = &intel_limits_pineview_lvds;
 614                 else
 615                         limit = &intel_limits_pineview_sdvo;
 616         } else if (IS_CHERRYVIEW(dev)) {
 617                 limit = &intel_limits_chv;
 618         } else if (IS_VALLEYVIEW(dev)) {
 619                 limit = &intel_limits_vlv;
 620         } else if (!IS_GEN2(dev)) {
 621                 if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
 622                         limit = &intel_limits_i9xx_lvds;
 623                 else
 624                         limit = &intel_limits_i9xx_sdvo;
 625         } else {
 626                 if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
 627                         limit = &intel_limits_i8xx_lvds;
 628                 else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO))
 629                         limit = &intel_limits_i8xx_dvo;
 630                 else
 631                         limit = &intel_limits_i8xx_dac;
 632         }
 633         return limit;
 634 }
 635
 636 /*
 637  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 638  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 639  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 640  * The helpers' return value is the rate of the clock that is fed to the
 641  * display engine's pipe which can be the above fast dot clock rate or a
 642  * divided-down version of it.
 643  */
 644 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 645 static int pnv_calc_dpll_params(int refclk, intel_clock_t *clock)
 646 {
 647         clock->m = clock->m2 + 2;
 648         clock->p = clock->p1 * clock->p2;
 649         if (WARN_ON(clock->n == 0 || clock->p == 0))
 650                 return 0;
 651         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 652         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 653
 654         return clock->dot;
 655 }
 656
 657 static uint32_t i9xx_dpll_compute_m(struct dpll *dpll)
 658 {
 659         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 660 }
 661
 662 static int i9xx_calc_dpll_params(int refclk, intel_clock_t *clock)
 663 {
 664         clock->m = i9xx_dpll_compute_m(clock);
 665         clock->p = clock->p1 * clock->p2;
 666         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 667                 return 0;
 668         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 669         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 670
 671         return clock->dot;
 672 }
 673
 674 static int vlv_calc_dpll_params(int refclk, intel_clock_t *clock)
 675 {
 676         clock->m = clock->m1 * clock->m2;
 677         clock->p = clock->p1 * clock->p2;
 678         if (WARN_ON(clock->n == 0 || clock->p == 0))
 679                 return 0;
 680         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 681         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 682
 683         return clock->dot / 5;
 684 }
 685
 686 int chv_calc_dpll_params(int refclk, intel_clock_t *clock)
 687 {
 688         clock->m = clock->m1 * clock->m2;
 689         clock->p = clock->p1 * clock->p2;
 690         if (WARN_ON(clock->n == 0 || clock->p == 0))
 691                 return 0;
 692         clock->vco = DIV_ROUND_CLOSEST_ULL((uint64_t)refclk * clock->m,
 693                         clock->n << 22);
 694         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 695
 696         return clock->dot / 5;
 697 }
 698
 699 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 700 /**
 701  * Returns whether the given set of divisors are valid for a given refclk with
 702  * the given connectors.
 703  */
 704
 705 static bool intel_PLL_is_valid(struct drm_device *dev,
 706                                const intel_limit_t *limit,
 707                                const intel_clock_t *clock)
 708 {
 709         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 710                 INTELPllInvalid("n out of range\n");
 711         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 712                 INTELPllInvalid("p1 out of range\n");
 713         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 714                 INTELPllInvalid("m2 out of range\n");
 715         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 716                 INTELPllInvalid("m1 out of range\n");
 717
 718         if (!IS_PINEVIEW(dev) && !IS_VALLEYVIEW(dev) && !IS_BROXTON(dev))
 719                 if (clock->m1 <= clock->m2)
 720                         INTELPllInvalid("m1 <= m2\n");
 721
 722         if (!IS_VALLEYVIEW(dev) && !IS_BROXTON(dev)) {
 723                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 724                         INTELPllInvalid("p out of range\n");
 725                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 726                         INTELPllInvalid("m out of range\n");
 727         }
 728
 729         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 730                 INTELPllInvalid("vco out of range\n");
 731         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 732          * connector, etc., rather than just a single range.
 733          */
 734         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 735                 INTELPllInvalid("dot out of range\n");
 736
 737         return true;
 738 }
 739
 740 static int
 741 i9xx_select_p2_div(const intel_limit_t *limit,
 742                    const struct intel_crtc_state *crtc_state,
 743                    int target)
 744 {
 745         struct drm_device *dev = crtc_state->base.crtc->dev;
 746
 747         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 748                 /*
 749                  * For LVDS just rely on its current settings for dual-channel.
 750                  * We haven't figured out how to reliably set up different
 751                  * single/dual channel state, if we even can.
 752                  */
 753                 if (intel_is_dual_link_lvds(dev))
 754                         return limit->p2.p2_fast;
 755                 else
 756                         return limit->p2.p2_slow;
 757         } else {
 758                 if (target < limit->p2.dot_limit)
 759                         return limit->p2.p2_slow;
 760                 else
 761                         return limit->p2.p2_fast;
 762         }
 763 }
 764
 765 static bool
 766 i9xx_find_best_dpll(const intel_limit_t *limit,
 767                     struct intel_crtc_state *crtc_state,
 768                     int target, int refclk, intel_clock_t *match_clock,
 769                     intel_clock_t *best_clock)
 770 {
 771         struct drm_device *dev = crtc_state->base.crtc->dev;
 772         intel_clock_t clock;
 773         int err = target;
 774
 775         memset(best_clock, 0, sizeof(*best_clock));
 776
 777         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 778
 779         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 780              clock.m1++) {
 781                 for (clock.m2 = limit->m2.min;
 782                      clock.m2 <= limit->m2.max; clock.m2++) {
 783                         if (clock.m2 >= clock.m1)
 784                                 break;
 785                         for (clock.n = limit->n.min;
 786                              clock.n <= limit->n.max; clock.n++) {
 787                                 for (clock.p1 = limit->p1.min;
 788                                         clock.p1 <= limit->p1.max; clock.p1++) {
 789                                         int this_err;
 790
 791                                         i9xx_calc_dpll_params(refclk, &clock);
 792                                         if (!intel_PLL_is_valid(dev, limit,
 793                                                                 &clock))
 794                                                 continue;
 795                                         if (match_clock &&
 796                                             clock.p != match_clock->p)
 797                                                 continue;
 798
 799                                         this_err = abs(clock.dot - target);
 800                                         if (this_err < err) {
 801                                                 *best_clock = clock;
 802                                                 err = this_err;
 803                                         }
 804                                 }
 805                         }
 806                 }
 807         }
 808
 809         return (err != target);
 810 }
 811
 812 static bool
 813 pnv_find_best_dpll(const intel_limit_t *limit,
 814                    struct intel_crtc_state *crtc_state,
 815                    int target, int refclk, intel_clock_t *match_clock,
 816                    intel_clock_t *best_clock)
 817 {
 818         struct drm_device *dev = crtc_state->base.crtc->dev;
 819         intel_clock_t clock;
 820         int err = target;
 821
 822         memset(best_clock, 0, sizeof(*best_clock));
 823
 824         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 825
 826         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 827              clock.m1++) {
 828                 for (clock.m2 = limit->m2.min;
 829                      clock.m2 <= limit->m2.max; clock.m2++) {
 830                         for (clock.n = limit->n.min;
 831                              clock.n <= limit->n.max; clock.n++) {
 832                                 for (clock.p1 = limit->p1.min;
 833                                         clock.p1 <= limit->p1.max; clock.p1++) {
 834                                         int this_err;
 835
 836                                         pnv_calc_dpll_params(refclk, &clock);
 837                                         if (!intel_PLL_is_valid(dev, limit,
 838                                                                 &clock))
 839                                                 continue;
 840                                         if (match_clock &&
 841                                             clock.p != match_clock->p)
 842                                                 continue;
 843
 844                                         this_err = abs(clock.dot - target);
 845                                         if (this_err < err) {
 846                                                 *best_clock = clock;
 847                                                 err = this_err;
 848                                         }
 849                                 }
 850                         }
 851                 }
 852         }
 853
 854         return (err != target);
 855 }
 856
 857 static bool
 858 g4x_find_best_dpll(const intel_limit_t *limit,
 859                    struct intel_crtc_state *crtc_state,
 860                    int target, int refclk, intel_clock_t *match_clock,
 861                    intel_clock_t *best_clock)
 862 {
 863         struct drm_device *dev = crtc_state->base.crtc->dev;
 864         intel_clock_t clock;
 865         int max_n;
 866         bool found = false;
 867         /* approximately equals target * 0.00585 */
 868         int err_most = (target >> 8) + (target >> 9);
 869
 870         memset(best_clock, 0, sizeof(*best_clock));
 871
 872         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 873
 874         max_n = limit->n.max;
 875         /* based on hardware requirement, prefer smaller n to precision */
 876         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 877                 /* based on hardware requirement, prefere larger m1,m2 */
 878                 for (clock.m1 = limit->m1.max;
 879                      clock.m1 >= limit->m1.min; clock.m1--) {
 880                         for (clock.m2 = limit->m2.max;
 881                              clock.m2 >= limit->m2.min; clock.m2--) {
 882                                 for (clock.p1 = limit->p1.max;
 883                                      clock.p1 >= limit->p1.min; clock.p1--) {
 884                                         int this_err;
 885
 886                                         i9xx_calc_dpll_params(refclk, &clock);
 887                                         if (!intel_PLL_is_valid(dev, limit,
 888                                                                 &clock))
 889                                                 continue;
 890
 891                                         this_err = abs(clock.dot - target);
 892                                         if (this_err < err_most) {
 893                                                 *best_clock = clock;
 894                                                 err_most = this_err;
 895                                                 max_n = clock.n;
 896                                                 found = true;
 897                                         }
 898                                 }
 899                         }
 900                 }
 901         }
 902         return found;
 903 }
 904
 905 /*
 906  * Check if the calculated PLL configuration is more optimal compared to the
 907  * best configuration and error found so far. Return the calculated error.
 908  */
 909 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 910                                const intel_clock_t *calculated_clock,
 911                                const intel_clock_t *best_clock,
 912                                unsigned int best_error_ppm,
 913                                unsigned int *error_ppm)
 914 {
 915         /*
 916          * For CHV ignore the error and consider only the P value.
 917          * Prefer a bigger P value based on HW requirements.
 918          */
 919         if (IS_CHERRYVIEW(dev)) {
 920                 *error_ppm = 0;
 921
 922                 return calculated_clock->p > best_clock->p;
 923         }
 924
 925         if (WARN_ON_ONCE(!target_freq))
 926                 return false;
 927
 928         *error_ppm = div_u64(1000000ULL *
 929                                 abs(target_freq - calculated_clock->dot),
 930                              target_freq);
 931         /*
 932          * Prefer a better P value over a better (smaller) error if the error
 933          * is small. Ensure this preference for future configurations too by
 934          * setting the error to 0.
 935          */
 936         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 937                 *error_ppm = 0;
 938
 939                 return true;
 940         }
 941
 942         return *error_ppm + 10 < best_error_ppm;
 943 }
 944
 945 static bool
 946 vlv_find_best_dpll(const intel_limit_t *limit,
 947                    struct intel_crtc_state *crtc_state,
 948                    int target, int refclk, intel_clock_t *match_clock,
 949                    intel_clock_t *best_clock)
 950 {
 951         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 952         struct drm_device *dev = crtc->base.dev;
 953         intel_clock_t clock;
 954         unsigned int bestppm = 1000000;
 955         /* min update 19.2 MHz */
 956         int max_n = min(limit->n.max, refclk / 19200);
 957         bool found = false;
 958
 959         target *= 5; /* fast clock */
 960
 961         memset(best_clock, 0, sizeof(*best_clock));
 962
 963         /* based on hardware requirement, prefer smaller n to precision */
 964         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 965                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 966                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 967                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 968                                 clock.p = clock.p1 * clock.p2;
 969                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 970                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 971                                         unsigned int ppm;
 972
 973                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 974                                                                      refclk * clock.m1);
 975
 976                                         vlv_calc_dpll_params(refclk, &clock);
 977
 978                                         if (!intel_PLL_is_valid(dev, limit,
 979                                                                 &clock))
 980                                                 continue;
 981
 982                                         if (!vlv_PLL_is_optimal(dev, target,
 983                                                                 &clock,
 984                                                                 best_clock,
 985                                                                 bestppm, &ppm))
 986                                                 continue;
 987
 988                                         *best_clock = clock;
 989                                         bestppm = ppm;
 990                                         found = true;
 991                                 }
 992                         }
 993                 }
 994         }
 995
 996         return found;
 997 }
 998
 999 static bool
1000 chv_find_best_dpll(const intel_limit_t *limit,
1001                    struct intel_crtc_state *crtc_state,
1002                    int target, int refclk, intel_clock_t *match_clock,
1003                    intel_clock_t *best_clock)
1004 {
1005         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1006         struct drm_device *dev = crtc->base.dev;
1007         unsigned int best_error_ppm;
1008         intel_clock_t clock;
1009         uint64_t m2;
1010         int found = false;
1011
1012         memset(best_clock, 0, sizeof(*best_clock));
1013         best_error_ppm = 1000000;
1014
1015         /*
1016          * Based on hardware doc, the n always set to 1, and m1 always
1017          * set to 2.  If requires to support 200Mhz refclk, we need to
1018          * revisit this because n may not 1 anymore.
1019          */
1020         clock.n = 1, clock.m1 = 2;
1021         target *= 5;    /* fast clock */
1022
1023         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
1024                 for (clock.p2 = limit->p2.p2_fast;
1025                                 clock.p2 >= limit->p2.p2_slow;
1026                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
1027                         unsigned int error_ppm;
1028
1029                         clock.p = clock.p1 * clock.p2;
1030
1031                         m2 = DIV_ROUND_CLOSEST_ULL(((uint64_t)target * clock.p *
1032                                         clock.n) << 22, refclk * clock.m1);
1033
1034                         if (m2 > INT_MAX/clock.m1)
1035                                 continue;
1036
1037                         clock.m2 = m2;
1038
1039                         chv_calc_dpll_params(refclk, &clock);
1040
1041                         if (!intel_PLL_is_valid(dev, limit, &clock))
1042                                 continue;
1043
1044                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
1045                                                 best_error_ppm, &error_ppm))
1046                                 continue;
1047
1048                         *best_clock = clock;
1049                         best_error_ppm = error_ppm;
1050                         found = true;
1051                 }
1052         }
1053
1054         return found;
1055 }
1056
1057 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
1058                         intel_clock_t *best_clock)
1059 {
1060         int refclk = i9xx_get_refclk(crtc_state, 0);
1061
1062         return chv_find_best_dpll(intel_limit(crtc_state, refclk), crtc_state,
1063                                   target_clock, refclk, NULL, best_clock);
1064 }
1065
1066 bool intel_crtc_active(struct drm_crtc *crtc)
1067 {
1068         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1069
1070         /* Be paranoid as we can arrive here with only partial
1071          * state retrieved from the hardware during setup.
1072          *
1073          * We can ditch the adjusted_mode.crtc_clock check as soon
1074          * as Haswell has gained clock readout/fastboot support.
1075          *
1076          * We can ditch the crtc->primary->fb check as soon as we can
1077          * properly reconstruct framebuffers.
1078          *
1079          * FIXME: The intel_crtc->active here should be switched to
1080          * crtc->state->active once we have proper CRTC states wired up
1081          * for atomic.
1082          */
1083         return intel_crtc->active && crtc->primary->state->fb &&
1084                 intel_crtc->config->base.adjusted_mode.crtc_clock;
1085 }
1086
1087 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
1088                                              enum pipe pipe)
1089 {
1090         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1091         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1092
1093         return intel_crtc->config->cpu_transcoder;
1094 }
1095
1096 static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
1097 {
1098         struct drm_i915_private *dev_priv = dev->dev_private;
1099         u32 reg = PIPEDSL(pipe);
1100         u32 line1, line2;
1101         u32 line_mask;
1102
1103         if (IS_GEN2(dev))
1104                 line_mask = DSL_LINEMASK_GEN2;
1105         else
1106                 line_mask = DSL_LINEMASK_GEN3;
1107
1108         line1 = I915_READ(reg) & line_mask;
1109         msleep(5);
1110         line2 = I915_READ(reg) & line_mask;
1111
1112         return line1 == line2;
1113 }
1114
1115 /*
1116  * intel_wait_for_pipe_off - wait for pipe to turn off
1117  * @crtc: crtc whose pipe to wait for
1118  *
1119  * After disabling a pipe, we can't wait for vblank in the usual way,
1120  * spinning on the vblank interrupt status bit, since we won't actually
1121  * see an interrupt when the pipe is disabled.
1122  *
1123  * On Gen4 and above:
1124  *   wait for the pipe register state bit to turn off
1125  *
1126  * Otherwise:
1127  *   wait for the display line value to settle (it usually
1128  *   ends up stopping at the start of the next frame).
1129  *
1130  */
1131 static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
1132 {
1133         struct drm_device *dev = crtc->base.dev;
1134         struct drm_i915_private *dev_priv = dev->dev_private;
1135         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
1136         enum pipe pipe = crtc->pipe;
1137
1138         if (INTEL_INFO(dev)->gen >= 4) {
1139                 int reg = PIPECONF(cpu_transcoder);
1140
1141                 /* Wait for the Pipe State to go off */
1142                 if (wait_for((I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0,
1143                              100))
1144                         WARN(1, "pipe_off wait timed out\n");
1145         } else {
1146                 /* Wait for the display line to settle */
1147                 if (wait_for(pipe_dsl_stopped(dev, pipe), 100))
1148                         WARN(1, "pipe_off wait timed out\n");
1149         }
1150 }
1151
1152 static const char *state_string(bool enabled)
1153 {
1154         return enabled ? "on" : "off";
1155 }
1156
1157 /* Only for pre-ILK configs */
1158 void assert_pll(struct drm_i915_private *dev_priv,
1159                 enum pipe pipe, bool state)
1160 {
1161         u32 val;
1162         bool cur_state;
1163
1164         val = I915_READ(DPLL(pipe));
1165         cur_state = !!(val & DPLL_VCO_ENABLE);
1166         I915_STATE_WARN(cur_state != state,
1167              "PLL state assertion failure (expected %s, current %s)\n",
1168              state_string(state), state_string(cur_state));
1169 }
1170
1171 /* XXX: the dsi pll is shared between MIPI DSI ports */
1172 static void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1173 {
1174         u32 val;
1175         bool cur_state;
1176
1177         mutex_lock(&dev_priv->sb_lock);
1178         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1179         mutex_unlock(&dev_priv->sb_lock);
1180
1181         cur_state = val & DSI_PLL_VCO_EN;
1182         I915_STATE_WARN(cur_state != state,
1183              "DSI PLL state assertion failure (expected %s, current %s)\n",
1184              state_string(state), state_string(cur_state));
1185 }
1186 #define assert_dsi_pll_enabled(d) assert_dsi_pll(d, true)
1187 #define assert_dsi_pll_disabled(d) assert_dsi_pll(d, false)
1188
1189 struct intel_shared_dpll *
1190 intel_crtc_to_shared_dpll(struct intel_crtc *crtc)
1191 {
1192         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
1193
1194         if (crtc->config->shared_dpll < 0)
1195                 return NULL;
1196
1197         return &dev_priv->shared_dplls[crtc->config->shared_dpll];
1198 }
1199
1200 /* For ILK+ */
1201 void assert_shared_dpll(struct drm_i915_private *dev_priv,
1202                         struct intel_shared_dpll *pll,
1203                         bool state)
1204 {
1205         bool cur_state;
1206         struct intel_dpll_hw_state hw_state;
1207
1208         if (WARN (!pll,
1209                   "asserting DPLL %s with no DPLL\n", state_string(state)))
1210                 return;
1211
1212         cur_state = pll->get_hw_state(dev_priv, pll, &hw_state);
1213         I915_STATE_WARN(cur_state != state,
1214              "%s assertion failure (expected %s, current %s)\n",
1215              pll->name, state_string(state), state_string(cur_state));
1216 }
1217
1218 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1219                           enum pipe pipe, bool state)
1220 {
1221         bool cur_state;
1222         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1223                                                                       pipe);
1224
1225         if (HAS_DDI(dev_priv->dev)) {
1226                 /* DDI does not have a specific FDI_TX register */
1227                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1228                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1229         } else {
1230                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1231                 cur_state = !!(val & FDI_TX_ENABLE);
1232         }
1233         I915_STATE_WARN(cur_state != state,
1234              "FDI TX state assertion failure (expected %s, current %s)\n",
1235              state_string(state), state_string(cur_state));
1236 }
1237 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1238 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1239
1240 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1241                           enum pipe pipe, bool state)
1242 {
1243         u32 val;
1244         bool cur_state;
1245
1246         val = I915_READ(FDI_RX_CTL(pipe));
1247         cur_state = !!(val & FDI_RX_ENABLE);
1248         I915_STATE_WARN(cur_state != state,
1249              "FDI RX state assertion failure (expected %s, current %s)\n",
1250              state_string(state), state_string(cur_state));
1251 }
1252 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1253 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1254
1255 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1256                                       enum pipe pipe)
1257 {
1258         u32 val;
1259
1260         /* ILK FDI PLL is always enabled */
1261         if (INTEL_INFO(dev_priv->dev)->gen == 5)
1262                 return;
1263
1264         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1265         if (HAS_DDI(dev_priv->dev))
1266                 return;
1267
1268         val = I915_READ(FDI_TX_CTL(pipe));
1269         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1270 }
1271
1272 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1273                        enum pipe pipe, bool state)
1274 {
1275         u32 val;
1276         bool cur_state;
1277
1278         val = I915_READ(FDI_RX_CTL(pipe));
1279         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1280         I915_STATE_WARN(cur_state != state,
1281              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1282              state_string(state), state_string(cur_state));
1283 }
1284
1285 void assert_panel_unlocked(struct drm_i915_private *dev_priv,
1286                            enum pipe pipe)
1287 {
1288         struct drm_device *dev = dev_priv->dev;
1289         int pp_reg;
1290         u32 val;
1291         enum pipe panel_pipe = PIPE_A;
1292         bool locked = true;
1293
1294         if (WARN_ON(HAS_DDI(dev)))
1295                 return;
1296
1297         if (HAS_PCH_SPLIT(dev)) {
1298                 u32 port_sel;
1299
1300                 pp_reg = PCH_PP_CONTROL;
1301                 port_sel = I915_READ(PCH_PP_ON_DELAYS) & PANEL_PORT_SELECT_MASK;
1302
1303                 if (port_sel == PANEL_PORT_SELECT_LVDS &&
1304                     I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT)
1305                         panel_pipe = PIPE_B;
1306                 /* XXX: else fix for eDP */
1307         } else if (IS_VALLEYVIEW(dev)) {
1308                 /* presumably write lock depends on pipe, not port select */
1309                 pp_reg = VLV_PIPE_PP_CONTROL(pipe);
1310                 panel_pipe = pipe;
1311         } else {
1312                 pp_reg = PP_CONTROL;
1313                 if (I915_READ(LVDS) & LVDS_PIPEB_SELECT)
1314                         panel_pipe = PIPE_B;
1315         }
1316
1317         val = I915_READ(pp_reg);
1318         if (!(val & PANEL_POWER_ON) ||
1319             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1320                 locked = false;
1321
1322         I915_STATE_WARN(panel_pipe == pipe && locked,
1323              "panel assertion failure, pipe %c regs locked\n",
1324              pipe_name(pipe));
1325 }
1326
1327 static void assert_cursor(struct drm_i915_private *dev_priv,
1328                           enum pipe pipe, bool state)
1329 {
1330         struct drm_device *dev = dev_priv->dev;
1331         bool cur_state;
1332
1333         if (IS_845G(dev) || IS_I865G(dev))
1334                 cur_state = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
1335         else
1336                 cur_state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE;
1337
1338         I915_STATE_WARN(cur_state != state,
1339              "cursor on pipe %c assertion failure (expected %s, current %s)\n",
1340              pipe_name(pipe), state_string(state), state_string(cur_state));
1341 }
1342 #define assert_cursor_enabled(d, p) assert_cursor(d, p, true)
1343 #define assert_cursor_disabled(d, p) assert_cursor(d, p, false)
1344
1345 void assert_pipe(struct drm_i915_private *dev_priv,
1346                  enum pipe pipe, bool state)
1347 {
1348         bool cur_state;
1349         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1350                                                                       pipe);
1351
1352         /* if we need the pipe quirk it must be always on */
1353         if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
1354             (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
1355                 state = true;
1356
1357         if (!intel_display_power_is_enabled(dev_priv,
1358                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder))) {
1359                 cur_state = false;
1360         } else {
1361                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1362                 cur_state = !!(val & PIPECONF_ENABLE);
1363         }
1364
1365         I915_STATE_WARN(cur_state != state,
1366              "pipe %c assertion failure (expected %s, current %s)\n",
1367              pipe_name(pipe), state_string(state), state_string(cur_state));
1368 }
1369
1370 static void assert_plane(struct drm_i915_private *dev_priv,
1371                          enum plane plane, bool state)
1372 {
1373         u32 val;
1374         bool cur_state;
1375
1376         val = I915_READ(DSPCNTR(plane));
1377         cur_state = !!(val & DISPLAY_PLANE_ENABLE);
1378         I915_STATE_WARN(cur_state != state,
1379              "plane %c assertion failure (expected %s, current %s)\n",
1380              plane_name(plane), state_string(state), state_string(cur_state));
1381 }
1382
1383 #define assert_plane_enabled(d, p) assert_plane(d, p, true)
1384 #define assert_plane_disabled(d, p) assert_plane(d, p, false)
1385
1386 static void assert_planes_disabled(struct drm_i915_private *dev_priv,
1387                                    enum pipe pipe)
1388 {
1389         struct drm_device *dev = dev_priv->dev;
1390         int i;
1391
1392         /* Primary planes are fixed to pipes on gen4+ */
1393         if (INTEL_INFO(dev)->gen >= 4) {
1394                 u32 val = I915_READ(DSPCNTR(pipe));
1395                 I915_STATE_WARN(val & DISPLAY_PLANE_ENABLE,
1396                      "plane %c assertion failure, should be disabled but not\n",
1397                      plane_name(pipe));
1398                 return;
1399         }
1400
1401         /* Need to check both planes against the pipe */
1402         for_each_pipe(dev_priv, i) {
1403                 u32 val = I915_READ(DSPCNTR(i));
1404                 enum pipe cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
1405                         DISPPLANE_SEL_PIPE_SHIFT;
1406                 I915_STATE_WARN((val & DISPLAY_PLANE_ENABLE) && pipe == cur_pipe,
1407                      "plane %c assertion failure, should be off on pipe %c but is still active\n",
1408                      plane_name(i), pipe_name(pipe));
1409         }
1410 }
1411
1412 static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
1413                                     enum pipe pipe)
1414 {
1415         struct drm_device *dev = dev_priv->dev;
1416         int sprite;
1417
1418         if (INTEL_INFO(dev)->gen >= 9) {
1419                 for_each_sprite(dev_priv, pipe, sprite) {
1420                         u32 val = I915_READ(PLANE_CTL(pipe, sprite));
1421                         I915_STATE_WARN(val & PLANE_CTL_ENABLE,
1422                              "plane %d assertion failure, should be off on pipe %c but is still active\n",
1423                              sprite, pipe_name(pipe));
1424                 }
1425         } else if (IS_VALLEYVIEW(dev)) {
1426                 for_each_sprite(dev_priv, pipe, sprite) {
1427                         u32 val = I915_READ(SPCNTR(pipe, sprite));
1428                         I915_STATE_WARN(val & SP_ENABLE,
1429                              "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1430                              sprite_name(pipe, sprite), pipe_name(pipe));
1431                 }
1432         } else if (INTEL_INFO(dev)->gen >= 7) {
1433                 u32 val = I915_READ(SPRCTL(pipe));
1434                 I915_STATE_WARN(val & SPRITE_ENABLE,
1435                      "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1436                      plane_name(pipe), pipe_name(pipe));
1437         } else if (INTEL_INFO(dev)->gen >= 5) {
1438                 u32 val = I915_READ(DVSCNTR(pipe));
1439                 I915_STATE_WARN(val & DVS_ENABLE,
1440                      "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1441                      plane_name(pipe), pipe_name(pipe));
1442         }
1443 }
1444
1445 static void assert_vblank_disabled(struct drm_crtc *crtc)
1446 {
1447         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1448                 drm_crtc_vblank_put(crtc);
1449 }
1450
1451 static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
1452 {
1453         u32 val;
1454         bool enabled;
1455
1456         I915_STATE_WARN_ON(!(HAS_PCH_IBX(dev_priv->dev) || HAS_PCH_CPT(dev_priv->dev)));
1457
1458         val = I915_READ(PCH_DREF_CONTROL);
1459         enabled = !!(val & (DREF_SSC_SOURCE_MASK | DREF_NONSPREAD_SOURCE_MASK |
1460                             DREF_SUPERSPREAD_SOURCE_MASK));
1461         I915_STATE_WARN(!enabled, "PCH refclk assertion failure, should be active but is disabled\n");
1462 }
1463
1464 static void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1465                                            enum pipe pipe)
1466 {
1467         u32 val;
1468         bool enabled;
1469
1470         val = I915_READ(PCH_TRANSCONF(pipe));
1471         enabled = !!(val & TRANS_ENABLE);
1472         I915_STATE_WARN(enabled,
1473              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1474              pipe_name(pipe));
1475 }
1476
1477 static bool dp_pipe_enabled(struct drm_i915_private *dev_priv,
1478                             enum pipe pipe, u32 port_sel, u32 val)
1479 {
1480         if ((val & DP_PORT_EN) == 0)
1481                 return false;
1482
1483         if (HAS_PCH_CPT(dev_priv->dev)) {
1484                 u32     trans_dp_ctl_reg = TRANS_DP_CTL(pipe);
1485                 u32     trans_dp_ctl = I915_READ(trans_dp_ctl_reg);
1486                 if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel)
1487                         return false;
1488         } else if (IS_CHERRYVIEW(dev_priv->dev)) {
1489                 if ((val & DP_PIPE_MASK_CHV) != DP_PIPE_SELECT_CHV(pipe))
1490                         return false;
1491         } else {
1492                 if ((val & DP_PIPE_MASK) != (pipe << 30))
1493                         return false;
1494         }
1495         return true;
1496 }
1497
1498 static bool hdmi_pipe_enabled(struct drm_i915_private *dev_priv,
1499                               enum pipe pipe, u32 val)
1500 {
1501         if ((val & SDVO_ENABLE) == 0)
1502                 return false;
1503
1504         if (HAS_PCH_CPT(dev_priv->dev)) {
1505                 if ((val & SDVO_PIPE_SEL_MASK_CPT) != SDVO_PIPE_SEL_CPT(pipe))
1506                         return false;
1507         } else if (IS_CHERRYVIEW(dev_priv->dev)) {
1508                 if ((val & SDVO_PIPE_SEL_MASK_CHV) != SDVO_PIPE_SEL_CHV(pipe))
1509                         return false;
1510         } else {
1511                 if ((val & SDVO_PIPE_SEL_MASK) != SDVO_PIPE_SEL(pipe))
1512                         return false;
1513         }
1514         return true;
1515 }
1516
1517 static bool lvds_pipe_enabled(struct drm_i915_private *dev_priv,
1518                               enum pipe pipe, u32 val)
1519 {
1520         if ((val & LVDS_PORT_EN) == 0)
1521                 return false;
1522
1523         if (HAS_PCH_CPT(dev_priv->dev)) {
1524                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1525                         return false;
1526         } else {
1527                 if ((val & LVDS_PIPE_MASK) != LVDS_PIPE(pipe))
1528                         return false;
1529         }
1530         return true;
1531 }
1532
1533 static bool adpa_pipe_enabled(struct drm_i915_private *dev_priv,
1534                               enum pipe pipe, u32 val)
1535 {
1536         if ((val & ADPA_DAC_ENABLE) == 0)
1537                 return false;
1538         if (HAS_PCH_CPT(dev_priv->dev)) {
1539                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1540                         return false;
1541         } else {
1542                 if ((val & ADPA_PIPE_SELECT_MASK) != ADPA_PIPE_SELECT(pipe))
1543                         return false;
1544         }
1545         return true;
1546 }
1547
1548 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1549                                    enum pipe pipe, int reg, u32 port_sel)
1550 {
1551         u32 val = I915_READ(reg);
1552         I915_STATE_WARN(dp_pipe_enabled(dev_priv, pipe, port_sel, val),
1553              "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
1554              reg, pipe_name(pipe));
1555
1556         I915_STATE_WARN(HAS_PCH_IBX(dev_priv->dev) && (val & DP_PORT_EN) == 0
1557              && (val & DP_PIPEB_SELECT),
1558              "IBX PCH dp port still using transcoder B\n");
1559 }
1560
1561 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1562                                      enum pipe pipe, int reg)
1563 {
1564         u32 val = I915_READ(reg);
1565         I915_STATE_WARN(hdmi_pipe_enabled(dev_priv, pipe, val),
1566              "PCH HDMI (0x%08x) enabled on transcoder %c, should be disabled\n",
1567              reg, pipe_name(pipe));
1568
1569         I915_STATE_WARN(HAS_PCH_IBX(dev_priv->dev) && (val & SDVO_ENABLE) == 0
1570              && (val & SDVO_PIPE_B_SELECT),
1571              "IBX PCH hdmi port still using transcoder B\n");
1572 }
1573
1574 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1575                                       enum pipe pipe)
1576 {
1577         u32 val;
1578
1579         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
1580         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
1581         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
1582
1583         val = I915_READ(PCH_ADPA);
1584         I915_STATE_WARN(adpa_pipe_enabled(dev_priv, pipe, val),
1585              "PCH VGA enabled on transcoder %c, should be disabled\n",
1586              pipe_name(pipe));
1587
1588         val = I915_READ(PCH_LVDS);
1589         I915_STATE_WARN(lvds_pipe_enabled(dev_priv, pipe, val),
1590              "PCH LVDS enabled on transcoder %c, should be disabled\n",
1591              pipe_name(pipe));
1592
1593         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMIB);
1594         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMIC);
1595         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMID);
1596 }
1597
1598 static void vlv_enable_pll(struct intel_crtc *crtc,
1599                            const struct intel_crtc_state *pipe_config)
1600 {
1601         struct drm_device *dev = crtc->base.dev;
1602         struct drm_i915_private *dev_priv = dev->dev_private;
1603         int reg = DPLL(crtc->pipe);
1604         u32 dpll = pipe_config->dpll_hw_state.dpll;
1605
1606         assert_pipe_disabled(dev_priv, crtc->pipe);
1607
1608         /* No really, not for ILK+ */
1609         BUG_ON(!IS_VALLEYVIEW(dev_priv->dev));
1610
1611         /* PLL is protected by panel, make sure we can write it */
1612         if (IS_MOBILE(dev_priv->dev))
1613                 assert_panel_unlocked(dev_priv, crtc->pipe);
1614
1615         I915_WRITE(reg, dpll);
1616         POSTING_READ(reg);
1617         udelay(150);
1618
1619         if (wait_for(((I915_READ(reg) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1))
1620                 DRM_ERROR("DPLL %d failed to lock\n", crtc->pipe);
1621
1622         I915_WRITE(DPLL_MD(crtc->pipe), pipe_config->dpll_hw_state.dpll_md);
1623         POSTING_READ(DPLL_MD(crtc->pipe));
1624
1625         /* We do this three times for luck */
1626         I915_WRITE(reg, dpll);
1627         POSTING_READ(reg);
1628         udelay(150); /* wait for warmup */
1629         I915_WRITE(reg, dpll);
1630         POSTING_READ(reg);
1631         udelay(150); /* wait for warmup */
1632         I915_WRITE(reg, dpll);
1633         POSTING_READ(reg);
1634         udelay(150); /* wait for warmup */
1635 }
1636
1637 static void chv_enable_pll(struct intel_crtc *crtc,
1638                            const struct intel_crtc_state *pipe_config)
1639 {
1640         struct drm_device *dev = crtc->base.dev;
1641         struct drm_i915_private *dev_priv = dev->dev_private;
1642         int pipe = crtc->pipe;
1643         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1644         u32 tmp;
1645
1646         assert_pipe_disabled(dev_priv, crtc->pipe);
1647
1648         BUG_ON(!IS_CHERRYVIEW(dev_priv->dev));
1649
1650         mutex_lock(&dev_priv->sb_lock);
1651
1652         /* Enable back the 10bit clock to display controller */
1653         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1654         tmp |= DPIO_DCLKP_EN;
1655         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1656
1657         mutex_unlock(&dev_priv->sb_lock);
1658
1659         /*
1660          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1661          */
1662         udelay(1);
1663
1664         /* Enable PLL */
1665         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1666
1667         /* Check PLL is locked */
1668         if (wait_for(((I915_READ(DPLL(pipe)) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1))
1669                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1670
1671         /* not sure when this should be written */
1672         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1673         POSTING_READ(DPLL_MD(pipe));
1674 }
1675
1676 static int intel_num_dvo_pipes(struct drm_device *dev)
1677 {
1678         struct intel_crtc *crtc;
1679         int count = 0;
1680
1681         for_each_intel_crtc(dev, crtc)
1682                 count += crtc->base.state->active &&
1683                         intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO);
1684
1685         return count;
1686 }
1687
1688 static void i9xx_enable_pll(struct intel_crtc *crtc)
1689 {
1690         struct drm_device *dev = crtc->base.dev;
1691         struct drm_i915_private *dev_priv = dev->dev_private;
1692         int reg = DPLL(crtc->pipe);
1693         u32 dpll = crtc->config->dpll_hw_state.dpll;
1694
1695         assert_pipe_disabled(dev_priv, crtc->pipe);
1696
1697         /* No really, not for ILK+ */
1698         BUG_ON(INTEL_INFO(dev)->gen >= 5);
1699
1700         /* PLL is protected by panel, make sure we can write it */
1701         if (IS_MOBILE(dev) && !IS_I830(dev))
1702                 assert_panel_unlocked(dev_priv, crtc->pipe);
1703
1704         /* Enable DVO 2x clock on both PLLs if necessary */
1705         if (IS_I830(dev) && intel_num_dvo_pipes(dev) > 0) {
1706                 /*
1707                  * It appears to be important that we don't enable this
1708                  * for the current pipe before otherwise configuring the
1709                  * PLL. No idea how this should be handled if multiple
1710                  * DVO outputs are enabled simultaneosly.
1711                  */
1712                 dpll |= DPLL_DVO_2X_MODE;
1713                 I915_WRITE(DPLL(!crtc->pipe),
1714                            I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE);
1715         }
1716
1717         /*
1718          * Apparently we need to have VGA mode enabled prior to changing
1719          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1720          * dividers, even though the register value does change.
1721          */
1722         I915_WRITE(reg, 0);
1723
1724         I915_WRITE(reg, dpll);
1725
1726         /* Wait for the clocks to stabilize. */
1727         POSTING_READ(reg);
1728         udelay(150);
1729
1730         if (INTEL_INFO(dev)->gen >= 4) {
1731                 I915_WRITE(DPLL_MD(crtc->pipe),
1732                            crtc->config->dpll_hw_state.dpll_md);
1733         } else {
1734                 /* The pixel multiplier can only be updated once the
1735                  * DPLL is enabled and the clocks are stable.
1736                  *
1737                  * So write it again.
1738                  */
1739                 I915_WRITE(reg, dpll);
1740         }
1741
1742         /* We do this three times for luck */
1743         I915_WRITE(reg, dpll);
1744         POSTING_READ(reg);
1745         udelay(150); /* wait for warmup */
1746         I915_WRITE(reg, dpll);
1747         POSTING_READ(reg);
1748         udelay(150); /* wait for warmup */
1749         I915_WRITE(reg, dpll);
1750         POSTING_READ(reg);
1751         udelay(150); /* wait for warmup */
1752 }
1753
1754 /**
1755  * i9xx_disable_pll - disable a PLL
1756  * @dev_priv: i915 private structure
1757  * @pipe: pipe PLL to disable
1758  *
1759  * Disable the PLL for @pipe, making sure the pipe is off first.
1760  *
1761  * Note!  This is for pre-ILK only.
1762  */
1763 static void i9xx_disable_pll(struct intel_crtc *crtc)
1764 {
1765         struct drm_device *dev = crtc->base.dev;
1766         struct drm_i915_private *dev_priv = dev->dev_private;
1767         enum pipe pipe = crtc->pipe;
1768
1769         /* Disable DVO 2x clock on both PLLs if necessary */
1770         if (IS_I830(dev) &&
1771             intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO) &&
1772             !intel_num_dvo_pipes(dev)) {
1773                 I915_WRITE(DPLL(PIPE_B),
1774                            I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
1775                 I915_WRITE(DPLL(PIPE_A),
1776                            I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE);
1777         }
1778
1779         /* Don't disable pipe or pipe PLLs if needed */
1780         if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
1781             (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
1782                 return;
1783
1784         /* Make sure the pipe isn't still relying on us */
1785         assert_pipe_disabled(dev_priv, pipe);
1786
1787         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1788         POSTING_READ(DPLL(pipe));
1789 }
1790
1791 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1792 {
1793         u32 val;
1794
1795         /* Make sure the pipe isn't still relying on us */
1796         assert_pipe_disabled(dev_priv, pipe);
1797
1798         /*
1799          * Leave integrated clock source and reference clock enabled for pipe B.
1800          * The latter is needed for VGA hotplug / manual detection.
1801          */
1802         val = DPLL_VGA_MODE_DIS;
1803         if (pipe == PIPE_B)
1804                 val = DPLL_INTEGRATED_CRI_CLK_VLV | DPLL_REF_CLK_ENABLE_VLV;
1805         I915_WRITE(DPLL(pipe), val);
1806         POSTING_READ(DPLL(pipe));
1807
1808 }
1809
1810 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1811 {
1812         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1813         u32 val;
1814
1815         /* Make sure the pipe isn't still relying on us */
1816         assert_pipe_disabled(dev_priv, pipe);
1817
1818         /* Set PLL en = 0 */
1819         val = DPLL_SSC_REF_CLK_CHV |
1820                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1821         if (pipe != PIPE_A)
1822                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1823         I915_WRITE(DPLL(pipe), val);
1824         POSTING_READ(DPLL(pipe));
1825
1826         mutex_lock(&dev_priv->sb_lock);
1827
1828         /* Disable 10bit clock to display controller */
1829         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1830         val &= ~DPIO_DCLKP_EN;
1831         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1832
1833         mutex_unlock(&dev_priv->sb_lock);
1834 }
1835
1836 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1837                          struct intel_digital_port *dport,
1838                          unsigned int expected_mask)
1839 {
1840         u32 port_mask;
1841         int dpll_reg;
1842
1843         switch (dport->port) {
1844         case PORT_B:
1845                 port_mask = DPLL_PORTB_READY_MASK;
1846                 dpll_reg = DPLL(0);
1847                 break;
1848         case PORT_C:
1849                 port_mask = DPLL_PORTC_READY_MASK;
1850                 dpll_reg = DPLL(0);
1851                 expected_mask <<= 4;
1852                 break;
1853         case PORT_D:
1854                 port_mask = DPLL_PORTD_READY_MASK;
1855                 dpll_reg = DPIO_PHY_STATUS;
1856                 break;
1857         default:
1858                 BUG();
1859         }
1860
1861         if (wait_for((I915_READ(dpll_reg) & port_mask) == expected_mask, 1000))
1862                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1863                      port_name(dport->port), I915_READ(dpll_reg) & port_mask, expected_mask);
1864 }
1865
1866 static void intel_prepare_shared_dpll(struct intel_crtc *crtc)
1867 {
1868         struct drm_device *dev = crtc->base.dev;
1869         struct drm_i915_private *dev_priv = dev->dev_private;
1870         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
1871
1872         if (WARN_ON(pll == NULL))
1873                 return;
1874
1875         WARN_ON(!pll->config.crtc_mask);
1876         if (pll->active == 0) {
1877                 DRM_DEBUG_DRIVER("setting up %s\n", pll->name);
1878                 WARN_ON(pll->on);
1879                 assert_shared_dpll_disabled(dev_priv, pll);
1880
1881                 pll->mode_set(dev_priv, pll);
1882         }
1883 }
1884
1885 /**
1886  * intel_enable_shared_dpll - enable PCH PLL
1887  * @dev_priv: i915 private structure
1888  * @pipe: pipe PLL to enable
1889  *
1890  * The PCH PLL needs to be enabled before the PCH transcoder, since it
1891  * drives the transcoder clock.
1892  */
1893 static void intel_enable_shared_dpll(struct intel_crtc *crtc)
1894 {
1895         struct drm_device *dev = crtc->base.dev;
1896         struct drm_i915_private *dev_priv = dev->dev_private;
1897         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
1898
1899         if (WARN_ON(pll == NULL))
1900                 return;
1901
1902         if (WARN_ON(pll->config.crtc_mask == 0))
1903                 return;
1904
1905         DRM_DEBUG_KMS("enable %s (active %d, on? %d) for crtc %d\n",
1906                       pll->name, pll->active, pll->on,
1907                       crtc->base.base.id);
1908
1909         if (pll->active++) {
1910                 WARN_ON(!pll->on);
1911                 assert_shared_dpll_enabled(dev_priv, pll);
1912                 return;
1913         }
1914         WARN_ON(pll->on);
1915
1916         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
1917
1918         DRM_DEBUG_KMS("enabling %s\n", pll->name);
1919         pll->enable(dev_priv, pll);
1920         pll->on = true;
1921 }
1922
1923 static void intel_disable_shared_dpll(struct intel_crtc *crtc)
1924 {
1925         struct drm_device *dev = crtc->base.dev;
1926         struct drm_i915_private *dev_priv = dev->dev_private;
1927         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
1928
1929         /* PCH only available on ILK+ */
1930         if (INTEL_INFO(dev)->gen < 5)
1931                 return;
1932
1933         if (pll == NULL)
1934                 return;
1935
1936         if (WARN_ON(!(pll->config.crtc_mask & (1 << drm_crtc_index(&crtc->base)))))
1937                 return;
1938
1939         DRM_DEBUG_KMS("disable %s (active %d, on? %d) for crtc %d\n",
1940                       pll->name, pll->active, pll->on,
1941                       crtc->base.base.id);
1942
1943         if (WARN_ON(pll->active == 0)) {
1944                 assert_shared_dpll_disabled(dev_priv, pll);
1945                 return;
1946         }
1947
1948         assert_shared_dpll_enabled(dev_priv, pll);
1949         WARN_ON(!pll->on);
1950         if (--pll->active)
1951                 return;
1952
1953         DRM_DEBUG_KMS("disabling %s\n", pll->name);
1954         pll->disable(dev_priv, pll);
1955         pll->on = false;
1956
1957         intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
1958 }
1959
1960 static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1961                                            enum pipe pipe)
1962 {
1963         struct drm_device *dev = dev_priv->dev;
1964         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1965         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1966         uint32_t reg, val, pipeconf_val;
1967
1968         /* PCH only available on ILK+ */
1969         BUG_ON(!HAS_PCH_SPLIT(dev));
1970
1971         /* Make sure PCH DPLL is enabled */
1972         assert_shared_dpll_enabled(dev_priv,
1973                                    intel_crtc_to_shared_dpll(intel_crtc));
1974
1975         /* FDI must be feeding us bits for PCH ports */
1976         assert_fdi_tx_enabled(dev_priv, pipe);
1977         assert_fdi_rx_enabled(dev_priv, pipe);
1978
1979         if (HAS_PCH_CPT(dev)) {
1980                 /* Workaround: Set the timing override bit before enabling the
1981                  * pch transcoder. */
1982                 reg = TRANS_CHICKEN2(pipe);
1983                 val = I915_READ(reg);
1984                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1985                 I915_WRITE(reg, val);
1986         }
1987
1988         reg = PCH_TRANSCONF(pipe);
1989         val = I915_READ(reg);
1990         pipeconf_val = I915_READ(PIPECONF(pipe));
1991
1992         if (HAS_PCH_IBX(dev_priv->dev)) {
1993                 /*
1994                  * Make the BPC in transcoder be consistent with
1995                  * that in pipeconf reg. For HDMI we must use 8bpc
1996                  * here for both 8bpc and 12bpc.
1997                  */
1998                 val &= ~PIPECONF_BPC_MASK;
1999                 if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_HDMI))
2000                         val |= PIPECONF_8BPC;
2001                 else
2002                         val |= pipeconf_val & PIPECONF_BPC_MASK;
2003         }
2004
2005         val &= ~TRANS_INTERLACE_MASK;
2006         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
2007                 if (HAS_PCH_IBX(dev_priv->dev) &&
2008                     intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
2009                         val |= TRANS_LEGACY_INTERLACED_ILK;
2010                 else
2011                         val |= TRANS_INTERLACED;
2012         else
2013                 val |= TRANS_PROGRESSIVE;
2014
2015         I915_WRITE(reg, val | TRANS_ENABLE);
2016         if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100))
2017                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
2018 }
2019
2020 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
2021                                       enum transcoder cpu_transcoder)
2022 {
2023         u32 val, pipeconf_val;
2024
2025         /* PCH only available on ILK+ */
2026         BUG_ON(!HAS_PCH_SPLIT(dev_priv->dev));
2027
2028         /* FDI must be feeding us bits for PCH ports */
2029         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
2030         assert_fdi_rx_enabled(dev_priv, TRANSCODER_A);
2031
2032         /* Workaround: set timing override bit. */
2033         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
2034         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
2035         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
2036
2037         val = TRANS_ENABLE;
2038         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
2039
2040         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
2041             PIPECONF_INTERLACED_ILK)
2042                 val |= TRANS_INTERLACED;
2043         else
2044                 val |= TRANS_PROGRESSIVE;
2045
2046         I915_WRITE(LPT_TRANSCONF, val);
2047         if (wait_for(I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE, 100))
2048                 DRM_ERROR("Failed to enable PCH transcoder\n");
2049 }
2050
2051 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
2052                                             enum pipe pipe)
2053 {
2054         struct drm_device *dev = dev_priv->dev;
2055         uint32_t reg, val;
2056
2057         /* FDI relies on the transcoder */
2058         assert_fdi_tx_disabled(dev_priv, pipe);
2059         assert_fdi_rx_disabled(dev_priv, pipe);
2060
2061         /* Ports must be off as well */
2062         assert_pch_ports_disabled(dev_priv, pipe);
2063
2064         reg = PCH_TRANSCONF(pipe);
2065         val = I915_READ(reg);
2066         val &= ~TRANS_ENABLE;
2067         I915_WRITE(reg, val);
2068         /* wait for PCH transcoder off, transcoder state */
2069         if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50))
2070                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
2071
2072         if (!HAS_PCH_IBX(dev)) {
2073                 /* Workaround: Clear the timing override chicken bit again. */
2074                 reg = TRANS_CHICKEN2(pipe);
2075                 val = I915_READ(reg);
2076                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
2077                 I915_WRITE(reg, val);
2078         }
2079 }
2080
2081 static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
2082 {
2083         u32 val;
2084
2085         val = I915_READ(LPT_TRANSCONF);
2086         val &= ~TRANS_ENABLE;
2087         I915_WRITE(LPT_TRANSCONF, val);
2088         /* wait for PCH transcoder off, transcoder state */
2089         if (wait_for((I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE) == 0, 50))
2090                 DRM_ERROR("Failed to disable PCH transcoder\n");
2091
2092         /* Workaround: clear timing override bit. */
2093         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
2094         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
2095         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
2096 }
2097
2098 /**
2099  * intel_enable_pipe - enable a pipe, asserting requirements
2100  * @crtc: crtc responsible for the pipe
2101  *
2102  * Enable @crtc's pipe, making sure that various hardware specific requirements
2103  * are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc.
2104  */
2105 static void intel_enable_pipe(struct intel_crtc *crtc)
2106 {
2107         struct drm_device *dev = crtc->base.dev;
2108         struct drm_i915_private *dev_priv = dev->dev_private;
2109         enum pipe pipe = crtc->pipe;
2110         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
2111                                                                       pipe);
2112         enum pipe pch_transcoder;
2113         int reg;
2114         u32 val;
2115
2116         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
2117
2118         assert_planes_disabled(dev_priv, pipe);
2119         assert_cursor_disabled(dev_priv, pipe);
2120         assert_sprites_disabled(dev_priv, pipe);
2121
2122         if (HAS_PCH_LPT(dev_priv->dev))
2123                 pch_transcoder = TRANSCODER_A;
2124         else
2125                 pch_transcoder = pipe;
2126
2127         /*
2128          * A pipe without a PLL won't actually be able to drive bits from
2129          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
2130          * need the check.
2131          */
2132         if (HAS_GMCH_DISPLAY(dev_priv->dev))
2133                 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI))
2134                         assert_dsi_pll_enabled(dev_priv);
2135                 else
2136                         assert_pll_enabled(dev_priv, pipe);
2137         else {
2138                 if (crtc->config->has_pch_encoder) {
2139                         /* if driving the PCH, we need FDI enabled */
2140                         assert_fdi_rx_pll_enabled(dev_priv, pch_transcoder);
2141                         assert_fdi_tx_pll_enabled(dev_priv,
2142                                                   (enum pipe) cpu_transcoder);
2143                 }
2144                 /* FIXME: assert CPU port conditions for SNB+ */
2145         }
2146
2147         reg = PIPECONF(cpu_transcoder);
2148         val = I915_READ(reg);
2149         if (val & PIPECONF_ENABLE) {
2150                 WARN_ON(!((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
2151                           (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)));
2152                 return;
2153         }
2154
2155         I915_WRITE(reg, val | PIPECONF_ENABLE);
2156         POSTING_READ(reg);
2157 }
2158
2159 /**
2160  * intel_disable_pipe - disable a pipe, asserting requirements
2161  * @crtc: crtc whose pipes is to be disabled
2162  *
2163  * Disable the pipe of @crtc, making sure that various hardware
2164  * specific requirements are met, if applicable, e.g. plane
2165  * disabled, panel fitter off, etc.
2166  *
2167  * Will wait until the pipe has shut down before returning.
2168  */
2169 static void intel_disable_pipe(struct intel_crtc *crtc)
2170 {
2171         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
2172         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
2173         enum pipe pipe = crtc->pipe;
2174         int reg;
2175         u32 val;
2176
2177         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
2178
2179         /*
2180          * Make sure planes won't keep trying to pump pixels to us,
2181          * or we might hang the display.
2182          */
2183         assert_planes_disabled(dev_priv, pipe);
2184         assert_cursor_disabled(dev_priv, pipe);
2185         assert_sprites_disabled(dev_priv, pipe);
2186
2187         reg = PIPECONF(cpu_transcoder);
2188         val = I915_READ(reg);
2189         if ((val & PIPECONF_ENABLE) == 0)
2190                 return;
2191
2192         /*
2193          * Double wide has implications for planes
2194          * so best keep it disabled when not needed.
2195          */
2196         if (crtc->config->double_wide)
2197                 val &= ~PIPECONF_DOUBLE_WIDE;
2198
2199         /* Don't disable pipe or pipe PLLs if needed */
2200         if (!(pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) &&
2201             !(pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
2202                 val &= ~PIPECONF_ENABLE;
2203
2204         I915_WRITE(reg, val);
2205         if ((val & PIPECONF_ENABLE) == 0)
2206                 intel_wait_for_pipe_off(crtc);
2207 }
2208
2209 static bool need_vtd_wa(struct drm_device *dev)
2210 {
2211 #ifdef CONFIG_INTEL_IOMMU
2212         if (INTEL_INFO(dev)->gen >= 6 && intel_iommu_gfx_mapped)
2213                 return true;
2214 #endif
2215         return false;
2216 }
2217
2218 unsigned int
2219 intel_tile_height(struct drm_device *dev, uint32_t pixel_format,
2220                   uint64_t fb_format_modifier, unsigned int plane)
2221 {
2222         unsigned int tile_height;
2223         uint32_t pixel_bytes;
2224
2225         switch (fb_format_modifier) {
2226         case DRM_FORMAT_MOD_NONE:
2227                 tile_height = 1;
2228                 break;
2229         case I915_FORMAT_MOD_X_TILED:
2230                 tile_height = IS_GEN2(dev) ? 16 : 8;
2231                 break;
2232         case I915_FORMAT_MOD_Y_TILED:
2233                 tile_height = 32;
2234                 break;
2235         case I915_FORMAT_MOD_Yf_TILED:
2236                 pixel_bytes = drm_format_plane_cpp(pixel_format, plane);
2237                 switch (pixel_bytes) {
2238                 default:
2239                 case 1:
2240                         tile_height = 64;
2241                         break;
2242                 case 2:
2243                 case 4:
2244                         tile_height = 32;
2245                         break;
2246                 case 8:
2247                         tile_height = 16;
2248                         break;
2249                 case 16:
2250                         WARN_ONCE(1,
2251                                   "128-bit pixels are not supported for display!");
2252                         tile_height = 16;
2253                         break;
2254                 }
2255                 break;
2256         default:
2257                 MISSING_CASE(fb_format_modifier);
2258                 tile_height = 1;
2259                 break;
2260         }
2261
2262         return tile_height;
2263 }
2264
2265 unsigned int
2266 intel_fb_align_height(struct drm_device *dev, unsigned int height,
2267                       uint32_t pixel_format, uint64_t fb_format_modifier)
2268 {
2269         return ALIGN(height, intel_tile_height(dev, pixel_format,
2270                                                fb_format_modifier, 0));
2271 }
2272
2273 static int
2274 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view, struct drm_framebuffer *fb,
2275                         const struct drm_plane_state *plane_state)
2276 {
2277         struct intel_rotation_info *info = &view->rotation_info;
2278         unsigned int tile_height, tile_pitch;
2279
2280         *view = i915_ggtt_view_normal;
2281
2282         if (!plane_state)
2283                 return 0;
2284
2285         if (!intel_rotation_90_or_270(plane_state->rotation))
2286                 return 0;
2287
2288         *view = i915_ggtt_view_rotated;
2289
2290         info->height = fb->height;
2291         info->pixel_format = fb->pixel_format;
2292         info->pitch = fb->pitches[0];
2293         info->uv_offset = fb->offsets[1];
2294         info->fb_modifier = fb->modifier[0];
2295
2296         tile_height = intel_tile_height(fb->dev, fb->pixel_format,
2297                                         fb->modifier[0], 0);
2298         tile_pitch = PAGE_SIZE / tile_height;
2299         info->width_pages = DIV_ROUND_UP(fb->pitches[0], tile_pitch);
2300         info->height_pages = DIV_ROUND_UP(fb->height, tile_height);
2301         info->size = info->width_pages * info->height_pages * PAGE_SIZE;
2302
2303         if (info->pixel_format == DRM_FORMAT_NV12) {
2304                 tile_height = intel_tile_height(fb->dev, fb->pixel_format,
2305                                                 fb->modifier[0], 1);
2306                 tile_pitch = PAGE_SIZE / tile_height;
2307                 info->width_pages_uv = DIV_ROUND_UP(fb->pitches[0], tile_pitch);
2308                 info->height_pages_uv = DIV_ROUND_UP(fb->height / 2,
2309                                                      tile_height);
2310                 info->size_uv = info->width_pages_uv * info->height_pages_uv *
2311                                 PAGE_SIZE;
2312         }
2313
2314         return 0;
2315 }
2316
2317 static unsigned int intel_linear_alignment(struct drm_i915_private *dev_priv)
2318 {
2319         if (INTEL_INFO(dev_priv)->gen >= 9)
2320                 return 256 * 1024;
2321         else if (IS_BROADWATER(dev_priv) || IS_CRESTLINE(dev_priv) ||
2322                  IS_VALLEYVIEW(dev_priv))
2323                 return 128 * 1024;
2324         else if (INTEL_INFO(dev_priv)->gen >= 4)
2325                 return 4 * 1024;
2326         else
2327                 return 0;
2328 }
2329
2330 int
2331 intel_pin_and_fence_fb_obj(struct drm_plane *plane,
2332                            struct drm_framebuffer *fb,
2333                            const struct drm_plane_state *plane_state,
2334                            struct intel_engine_cs *pipelined,
2335                            struct drm_i915_gem_request **pipelined_request)
2336 {
2337         struct drm_device *dev = fb->dev;
2338         struct drm_i915_private *dev_priv = dev->dev_private;
2339         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2340         struct i915_ggtt_view view;
2341         u32 alignment;
2342         int ret;
2343
2344         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2345
2346         switch (fb->modifier[0]) {
2347         case DRM_FORMAT_MOD_NONE:
2348                 alignment = intel_linear_alignment(dev_priv);
2349                 break;
2350         case I915_FORMAT_MOD_X_TILED:
2351                 if (INTEL_INFO(dev)->gen >= 9)
2352                         alignment = 256 * 1024;
2353                 else {
2354                         /* pin() will align the object as required by fence */
2355                         alignment = 0;
2356                 }
2357                 break;
2358         case I915_FORMAT_MOD_Y_TILED:
2359         case I915_FORMAT_MOD_Yf_TILED:
2360                 if (WARN_ONCE(INTEL_INFO(dev)->gen < 9,
2361                           "Y tiling bo slipped through, driver bug!\n"))
2362                         return -EINVAL;
2363                 alignment = 1 * 1024 * 1024;
2364                 break;
2365         default:
2366                 MISSING_CASE(fb->modifier[0]);
2367                 return -EINVAL;
2368         }
2369
2370         ret = intel_fill_fb_ggtt_view(&view, fb, plane_state);
2371         if (ret)
2372                 return ret;
2373
2374         /* Note that the w/a also requires 64 PTE of padding following the
2375          * bo. We currently fill all unused PTE with the shadow page and so
2376          * we should always have valid PTE following the scanout preventing
2377          * the VT-d warning.
2378          */
2379         if (need_vtd_wa(dev) && alignment < 256 * 1024)
2380                 alignment = 256 * 1024;
2381
2382         /*
2383          * Global gtt pte registers are special registers which actually forward
2384          * writes to a chunk of system memory. Which means that there is no risk
2385          * that the register values disappear as soon as we call
2386          * intel_runtime_pm_put(), so it is correct to wrap only the
2387          * pin/unpin/fence and not more.
2388          */
2389         intel_runtime_pm_get(dev_priv);
2390
2391         dev_priv->mm.interruptible = false;
2392         ret = i915_gem_object_pin_to_display_plane(obj, alignment, pipelined,
2393                                                    pipelined_request, &view);
2394         if (ret)
2395                 goto err_interruptible;
2396
2397         /* Install a fence for tiled scan-out. Pre-i965 always needs a
2398          * fence, whereas 965+ only requires a fence if using
2399          * framebuffer compression.  For simplicity, we always install
2400          * a fence as the cost is not that onerous.
2401          */
2402         if (view.type == I915_GGTT_VIEW_NORMAL) {
2403                 ret = i915_gem_object_get_fence(obj);
2404                 if (ret == -EDEADLK) {
2405                         /*
2406                          * -EDEADLK means there are no free fences
2407                          * no pending flips.
2408                          *
2409                          * This is propagated to atomic, but it uses
2410                          * -EDEADLK to force a locking recovery, so
2411                          * change the returned error to -EBUSY.
2412                          */
2413                         ret = -EBUSY;
2414                         goto err_unpin;
2415                 } else if (ret)
2416                         goto err_unpin;
2417
2418                 i915_gem_object_pin_fence(obj);
2419         }
2420
2421         dev_priv->mm.interruptible = true;
2422         intel_runtime_pm_put(dev_priv);
2423         return 0;
2424
2425 err_unpin:
2426         i915_gem_object_unpin_from_display_plane(obj, &view);
2427 err_interruptible:
2428         dev_priv->mm.interruptible = true;
2429         intel_runtime_pm_put(dev_priv);
2430         return ret;
2431 }
2432
2433 static void intel_unpin_fb_obj(struct drm_framebuffer *fb,
2434                                const struct drm_plane_state *plane_state)
2435 {
2436         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2437         struct i915_ggtt_view view;
2438         int ret;
2439
2440         WARN_ON(!mutex_is_locked(&obj->base.dev->struct_mutex));
2441
2442         ret = intel_fill_fb_ggtt_view(&view, fb, plane_state);
2443         WARN_ONCE(ret, "Couldn't get view from plane state!");
2444
2445         if (view.type == I915_GGTT_VIEW_NORMAL)
2446                 i915_gem_object_unpin_fence(obj);
2447
2448         i915_gem_object_unpin_from_display_plane(obj, &view);
2449 }
2450
2451 /* Computes the linear offset to the base tile and adjusts x, y. bytes per pixel
2452  * is assumed to be a power-of-two. */
2453 unsigned long intel_gen4_compute_page_offset(struct drm_i915_private *dev_priv,
2454                                              int *x, int *y,
2455                                              unsigned int tiling_mode,
2456                                              unsigned int cpp,
2457                                              unsigned int pitch)
2458 {
2459         if (tiling_mode != I915_TILING_NONE) {
2460                 unsigned int tile_rows, tiles;
2461
2462                 tile_rows = *y / 8;
2463                 *y %= 8;
2464
2465                 tiles = *x / (512/cpp);
2466                 *x %= 512/cpp;
2467
2468                 return tile_rows * pitch * 8 + tiles * 4096;
2469         } else {
2470                 unsigned int alignment = intel_linear_alignment(dev_priv) - 1;
2471                 unsigned int offset;
2472
2473                 offset = *y * pitch + *x * cpp;
2474                 *y = (offset & alignment) / pitch;
2475                 *x = ((offset & alignment) - *y * pitch) / cpp;
2476                 return offset & ~alignment;
2477         }
2478 }
2479
2480 static int i9xx_format_to_fourcc(int format)
2481 {
2482         switch (format) {
2483         case DISPPLANE_8BPP:
2484                 return DRM_FORMAT_C8;
2485         case DISPPLANE_BGRX555:
2486                 return DRM_FORMAT_XRGB1555;
2487         case DISPPLANE_BGRX565:
2488                 return DRM_FORMAT_RGB565;
2489         default:
2490         case DISPPLANE_BGRX888:
2491                 return DRM_FORMAT_XRGB8888;
2492         case DISPPLANE_RGBX888:
2493                 return DRM_FORMAT_XBGR8888;
2494         case DISPPLANE_BGRX101010:
2495                 return DRM_FORMAT_XRGB2101010;
2496         case DISPPLANE_RGBX101010:
2497                 return DRM_FORMAT_XBGR2101010;
2498         }
2499 }
2500
2501 static int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2502 {
2503         switch (format) {
2504         case PLANE_CTL_FORMAT_RGB_565:
2505                 return DRM_FORMAT_RGB565;
2506         default:
2507         case PLANE_CTL_FORMAT_XRGB_8888:
2508                 if (rgb_order) {
2509                         if (alpha)
2510                                 return DRM_FORMAT_ABGR8888;
2511                         else
2512                                 return DRM_FORMAT_XBGR8888;
2513                 } else {
2514                         if (alpha)
2515                                 return DRM_FORMAT_ARGB8888;
2516                         else
2517                                 return DRM_FORMAT_XRGB8888;
2518                 }
2519         case PLANE_CTL_FORMAT_XRGB_2101010:
2520                 if (rgb_order)
2521                         return DRM_FORMAT_XBGR2101010;
2522                 else
2523                         return DRM_FORMAT_XRGB2101010;
2524         }
2525 }
2526
2527 static bool
2528 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
2529                               struct intel_initial_plane_config *plane_config)
2530 {
2531         struct drm_device *dev = crtc->base.dev;
2532         struct drm_i915_private *dev_priv = to_i915(dev);
2533         struct drm_i915_gem_object *obj = NULL;
2534         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
2535         struct drm_framebuffer *fb = &plane_config->fb->base;
2536         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
2537         u32 size_aligned = round_up(plane_config->base + plane_config->size,
2538                                     PAGE_SIZE);
2539
2540         size_aligned -= base_aligned;
2541
2542         if (plane_config->size == 0)
2543                 return false;
2544
2545         /* If the FB is too big, just don't use it since fbdev is not very
2546          * important and we should probably use that space with FBC or other
2547          * features. */
2548         if (size_aligned * 2 > dev_priv->gtt.stolen_usable_size)
2549                 return false;
2550
2551         obj = i915_gem_object_create_stolen_for_preallocated(dev,
2552                                                              base_aligned,
2553                                                              base_aligned,
2554                                                              size_aligned);
2555         if (!obj)
2556                 return false;
2557
2558         obj->tiling_mode = plane_config->tiling;
2559         if (obj->tiling_mode == I915_TILING_X)
2560                 obj->stride = fb->pitches[0];
2561
2562         mode_cmd.pixel_format = fb->pixel_format;
2563         mode_cmd.width = fb->width;
2564         mode_cmd.height = fb->height;
2565         mode_cmd.pitches[0] = fb->pitches[0];
2566         mode_cmd.modifier[0] = fb->modifier[0];
2567         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
2568
2569         mutex_lock(&dev->struct_mutex);
2570         if (intel_framebuffer_init(dev, to_intel_framebuffer(fb),
2571                                    &mode_cmd, obj)) {
2572                 DRM_DEBUG_KMS("intel fb init failed\n");
2573                 goto out_unref_obj;
2574         }
2575         mutex_unlock(&dev->struct_mutex);
2576
2577         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
2578         return true;
2579
2580 out_unref_obj:
2581         drm_gem_object_unreference(&obj->base);
2582         mutex_unlock(&dev->struct_mutex);
2583         return false;
2584 }
2585
2586 /* Update plane->state->fb to match plane->fb after driver-internal updates */
2587 static void
2588 update_state_fb(struct drm_plane *plane)
2589 {
2590         if (plane->fb == plane->state->fb)
2591                 return;
2592
2593         if (plane->state->fb)
2594                 drm_framebuffer_unreference(plane->state->fb);
2595         plane->state->fb = plane->fb;
2596         if (plane->state->fb)
2597                 drm_framebuffer_reference(plane->state->fb);
2598 }
2599
2600 static void
2601 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
2602                              struct intel_initial_plane_config *plane_config)
2603 {
2604         struct drm_device *dev = intel_crtc->base.dev;
2605         struct drm_i915_private *dev_priv = dev->dev_private;
2606         struct drm_crtc *c;
2607         struct intel_crtc *i;
2608         struct drm_i915_gem_object *obj;
2609         struct drm_plane *primary = intel_crtc->base.primary;
2610         struct drm_plane_state *plane_state = primary->state;
2611         struct drm_crtc_state *crtc_state = intel_crtc->base.state;
2612         struct intel_plane *intel_plane = to_intel_plane(primary);
2613         struct drm_framebuffer *fb;
2614
2615         if (!plane_config->fb)
2616                 return;
2617
2618         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
2619                 fb = &plane_config->fb->base;
2620                 goto valid_fb;
2621         }
2622
2623         kfree(plane_config->fb);
2624
2625         /*
2626          * Failed to alloc the obj, check to see if we should share
2627          * an fb with another CRTC instead
2628          */
2629         for_each_crtc(dev, c) {
2630                 i = to_intel_crtc(c);
2631
2632                 if (c == &intel_crtc->base)
2633                         continue;
2634
2635                 if (!i->active)
2636                         continue;
2637
2638                 fb = c->primary->fb;
2639                 if (!fb)
2640                         continue;
2641
2642                 obj = intel_fb_obj(fb);
2643                 if (i915_gem_obj_ggtt_offset(obj) == plane_config->base) {
2644                         drm_framebuffer_reference(fb);
2645                         goto valid_fb;
2646                 }
2647         }
2648
2649         /*
2650          * We've failed to reconstruct the BIOS FB.  Current display state
2651          * indicates that the primary plane is visible, but has a NULL FB,
2652          * which will lead to problems later if we don't fix it up.  The
2653          * simplest solution is to just disable the primary plane now and
2654          * pretend the BIOS never had it enabled.
2655          */
2656         to_intel_plane_state(plane_state)->visible = false;
2657         crtc_state->plane_mask &= ~(1 << drm_plane_index(primary));
2658         intel_pre_disable_primary(&intel_crtc->base);
2659         intel_plane->disable_plane(primary, &intel_crtc->base);
2660
2661         return;
2662
2663 valid_fb:
2664         plane_state->src_x = 0;
2665         plane_state->src_y = 0;
2666         plane_state->src_w = fb->width << 16;
2667         plane_state->src_h = fb->height << 16;
2668
2669         plane_state->crtc_x = 0;
2670         plane_state->crtc_y = 0;
2671         plane_state->crtc_w = fb->width;
2672         plane_state->crtc_h = fb->height;
2673
2674         obj = intel_fb_obj(fb);
2675         if (obj->tiling_mode != I915_TILING_NONE)
2676                 dev_priv->preserve_bios_swizzle = true;
2677
2678         drm_framebuffer_reference(fb);
2679         primary->fb = primary->state->fb = fb;
2680         primary->crtc = primary->state->crtc = &intel_crtc->base;
2681         intel_crtc->base.state->plane_mask |= (1 << drm_plane_index(primary));
2682         obj->frontbuffer_bits |= to_intel_plane(primary)->frontbuffer_bit;
2683 }
2684
2685 static void i9xx_update_primary_plane(struct drm_crtc *crtc,
2686                                       struct drm_framebuffer *fb,
2687                                       int x, int y)
2688 {
2689         struct drm_device *dev = crtc->dev;
2690         struct drm_i915_private *dev_priv = dev->dev_private;
2691         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2692         struct drm_plane *primary = crtc->primary;
2693         bool visible = to_intel_plane_state(primary->state)->visible;
2694         struct drm_i915_gem_object *obj;
2695         int plane = intel_crtc->plane;
2696         unsigned long linear_offset;
2697         u32 dspcntr;
2698         u32 reg = DSPCNTR(plane);
2699         int pixel_size;
2700
2701         if (!visible || !fb) {
2702                 I915_WRITE(reg, 0);
2703                 if (INTEL_INFO(dev)->gen >= 4)
2704                         I915_WRITE(DSPSURF(plane), 0);
2705                 else
2706                         I915_WRITE(DSPADDR(plane), 0);
2707                 POSTING_READ(reg);
2708                 return;
2709         }
2710
2711         obj = intel_fb_obj(fb);
2712         if (WARN_ON(obj == NULL))
2713                 return;
2714
2715         pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
2716
2717         dspcntr = DISPPLANE_GAMMA_ENABLE;
2718
2719         dspcntr |= DISPLAY_PLANE_ENABLE;
2720
2721         if (INTEL_INFO(dev)->gen < 4) {
2722                 if (intel_crtc->pipe == PIPE_B)
2723                         dspcntr |= DISPPLANE_SEL_PIPE_B;
2724
2725                 /* pipesrc and dspsize control the size that is scaled from,
2726                  * which should always be the user's requested size.
2727                  */
2728                 I915_WRITE(DSPSIZE(plane),
2729                            ((intel_crtc->config->pipe_src_h - 1) << 16) |
2730                            (intel_crtc->config->pipe_src_w - 1));
2731                 I915_WRITE(DSPPOS(plane), 0);
2732         } else if (IS_CHERRYVIEW(dev) && plane == PLANE_B) {
2733                 I915_WRITE(PRIMSIZE(plane),
2734                            ((intel_crtc->config->pipe_src_h - 1) << 16) |
2735                            (intel_crtc->config->pipe_src_w - 1));
2736                 I915_WRITE(PRIMPOS(plane), 0);
2737                 I915_WRITE(PRIMCNSTALPHA(plane), 0);
2738         }
2739
2740         switch (fb->pixel_format) {
2741         case DRM_FORMAT_C8:
2742                 dspcntr |= DISPPLANE_8BPP;
2743                 break;
2744         case DRM_FORMAT_XRGB1555:
2745                 dspcntr |= DISPPLANE_BGRX555;
2746                 break;
2747         case DRM_FORMAT_RGB565:
2748                 dspcntr |= DISPPLANE_BGRX565;
2749                 break;
2750         case DRM_FORMAT_XRGB8888:
2751                 dspcntr |= DISPPLANE_BGRX888;
2752                 break;
2753         case DRM_FORMAT_XBGR8888:
2754                 dspcntr |= DISPPLANE_RGBX888;
2755                 break;
2756         case DRM_FORMAT_XRGB2101010:
2757                 dspcntr |= DISPPLANE_BGRX101010;
2758                 break;
2759         case DRM_FORMAT_XBGR2101010:
2760                 dspcntr |= DISPPLANE_RGBX101010;
2761                 break;
2762         default:
2763                 BUG();
2764         }
2765
2766         if (INTEL_INFO(dev)->gen >= 4 &&
2767             obj->tiling_mode != I915_TILING_NONE)
2768                 dspcntr |= DISPPLANE_TILED;
2769
2770         if (IS_G4X(dev))
2771                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
2772
2773         linear_offset = y * fb->pitches[0] + x * pixel_size;
2774
2775         if (INTEL_INFO(dev)->gen >= 4) {
2776                 intel_crtc->dspaddr_offset =
2777                         intel_gen4_compute_page_offset(dev_priv,
2778                                                        &x, &y, obj->tiling_mode,
2779                                                        pixel_size,
2780                                                        fb->pitches[0]);
2781                 linear_offset -= intel_crtc->dspaddr_offset;
2782         } else {
2783                 intel_crtc->dspaddr_offset = linear_offset;
2784         }
2785
2786         if (crtc->primary->state->rotation == BIT(DRM_ROTATE_180)) {
2787                 dspcntr |= DISPPLANE_ROTATE_180;
2788
2789                 x += (intel_crtc->config->pipe_src_w - 1);
2790                 y += (intel_crtc->config->pipe_src_h - 1);
2791
2792                 /* Finding the last pixel of the last line of the display
2793                 data and adding to linear_offset*/
2794                 linear_offset +=
2795                         (intel_crtc->config->pipe_src_h - 1) * fb->pitches[0] +
2796                         (intel_crtc->config->pipe_src_w - 1) * pixel_size;
2797         }
2798
2799         intel_crtc->adjusted_x = x;
2800         intel_crtc->adjusted_y = y;
2801
2802         I915_WRITE(reg, dspcntr);
2803
2804         I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
2805         if (INTEL_INFO(dev)->gen >= 4) {
2806                 I915_WRITE(DSPSURF(plane),
2807                            i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset);
2808                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2809                 I915_WRITE(DSPLINOFF(plane), linear_offset);
2810         } else
2811                 I915_WRITE(DSPADDR(plane), i915_gem_obj_ggtt_offset(obj) + linear_offset);
2812         POSTING_READ(reg);
2813 }
2814
2815 static void ironlake_update_primary_plane(struct drm_crtc *crtc,
2816                                           struct drm_framebuffer *fb,
2817                                           int x, int y)
2818 {
2819         struct drm_device *dev = crtc->dev;
2820         struct drm_i915_private *dev_priv = dev->dev_private;
2821         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2822         struct drm_plane *primary = crtc->primary;
2823         bool visible = to_intel_plane_state(primary->state)->visible;
2824         struct drm_i915_gem_object *obj;
2825         int plane = intel_crtc->plane;
2826         unsigned long linear_offset;
2827         u32 dspcntr;
2828         u32 reg = DSPCNTR(plane);
2829         int pixel_size;
2830
2831         if (!visible || !fb) {
2832                 I915_WRITE(reg, 0);
2833                 I915_WRITE(DSPSURF(plane), 0);
2834                 POSTING_READ(reg);
2835                 return;
2836         }
2837
2838         obj = intel_fb_obj(fb);
2839         if (WARN_ON(obj == NULL))
2840                 return;
2841
2842         pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
2843
2844         dspcntr = DISPPLANE_GAMMA_ENABLE;
2845
2846         dspcntr |= DISPLAY_PLANE_ENABLE;
2847
2848         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2849                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
2850
2851         switch (fb->pixel_format) {
2852         case DRM_FORMAT_C8:
2853                 dspcntr |= DISPPLANE_8BPP;
2854                 break;
2855         case DRM_FORMAT_RGB565:
2856                 dspcntr |= DISPPLANE_BGRX565;
2857                 break;
2858         case DRM_FORMAT_XRGB8888:
2859                 dspcntr |= DISPPLANE_BGRX888;
2860                 break;
2861         case DRM_FORMAT_XBGR8888:
2862                 dspcntr |= DISPPLANE_RGBX888;
2863                 break;
2864         case DRM_FORMAT_XRGB2101010:
2865                 dspcntr |= DISPPLANE_BGRX101010;
2866                 break;
2867         case DRM_FORMAT_XBGR2101010:
2868                 dspcntr |= DISPPLANE_RGBX101010;
2869                 break;
2870         default:
2871                 BUG();
2872         }
2873
2874         if (obj->tiling_mode != I915_TILING_NONE)
2875                 dspcntr |= DISPPLANE_TILED;
2876
2877         if (!IS_HASWELL(dev) && !IS_BROADWELL(dev))
2878                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
2879
2880         linear_offset = y * fb->pitches[0] + x * pixel_size;
2881         intel_crtc->dspaddr_offset =
2882                 intel_gen4_compute_page_offset(dev_priv,
2883                                                &x, &y, obj->tiling_mode,
2884                                                pixel_size,
2885                                                fb->pitches[0]);
2886         linear_offset -= intel_crtc->dspaddr_offset;
2887         if (crtc->primary->state->rotation == BIT(DRM_ROTATE_180)) {
2888                 dspcntr |= DISPPLANE_ROTATE_180;
2889
2890                 if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
2891                         x += (intel_crtc->config->pipe_src_w - 1);
2892                         y += (intel_crtc->config->pipe_src_h - 1);
2893
2894                         /* Finding the last pixel of the last line of the display
2895                         data and adding to linear_offset*/
2896                         linear_offset +=
2897                                 (intel_crtc->config->pipe_src_h - 1) * fb->pitches[0] +
2898                                 (intel_crtc->config->pipe_src_w - 1) * pixel_size;
2899                 }
2900         }
2901
2902         intel_crtc->adjusted_x = x;
2903         intel_crtc->adjusted_y = y;
2904
2905         I915_WRITE(reg, dspcntr);
2906
2907         I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
2908         I915_WRITE(DSPSURF(plane),
2909                    i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset);
2910         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2911                 I915_WRITE(DSPOFFSET(plane), (y << 16) | x);
2912         } else {
2913                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2914                 I915_WRITE(DSPLINOFF(plane), linear_offset);
2915         }
2916         POSTING_READ(reg);
2917 }
2918
2919 u32 intel_fb_stride_alignment(struct drm_device *dev, uint64_t fb_modifier,
2920                               uint32_t pixel_format)
2921 {
2922         u32 bits_per_pixel = drm_format_plane_cpp(pixel_format, 0) * 8;
2923
2924         /*
2925          * The stride is either expressed as a multiple of 64 bytes
2926          * chunks for linear buffers or in number of tiles for tiled
2927          * buffers.
2928          */
2929         switch (fb_modifier) {
2930         case DRM_FORMAT_MOD_NONE:
2931                 return 64;
2932         case I915_FORMAT_MOD_X_TILED:
2933                 if (INTEL_INFO(dev)->gen == 2)
2934                         return 128;
2935                 return 512;
2936         case I915_FORMAT_MOD_Y_TILED:
2937                 /* No need to check for old gens and Y tiling since this is
2938                  * about the display engine and those will be blocked before
2939                  * we get here.
2940                  */
2941                 return 128;
2942         case I915_FORMAT_MOD_Yf_TILED:
2943                 if (bits_per_pixel == 8)
2944                         return 64;
2945                 else
2946                         return 128;
2947         default:
2948                 MISSING_CASE(fb_modifier);
2949                 return 64;
2950         }
2951 }
2952
2953 unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane,
2954                                      struct drm_i915_gem_object *obj,
2955                                      unsigned int plane)
2956 {
2957         const struct i915_ggtt_view *view = &i915_ggtt_view_normal;
2958         struct i915_vma *vma;
2959         unsigned char *offset;
2960
2961         if (intel_rotation_90_or_270(intel_plane->base.state->rotation))
2962                 view = &i915_ggtt_view_rotated;
2963
2964         vma = i915_gem_obj_to_ggtt_view(obj, view);
2965         if (WARN(!vma, "ggtt vma for display object not found! (view=%u)\n",
2966                 view->type))
2967                 return -1;
2968
2969         offset = (unsigned char *)vma->node.start;
2970
2971         if (plane == 1) {
2972                 offset += vma->ggtt_view.rotation_info.uv_start_page *
2973                           PAGE_SIZE;
2974         }
2975
2976         return (unsigned long)offset;
2977 }
2978
2979 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
2980 {
2981         struct drm_device *dev = intel_crtc->base.dev;
2982         struct drm_i915_private *dev_priv = dev->dev_private;
2983
2984         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
2985         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
2986         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
2987 }
2988
2989 /*
2990  * This function detaches (aka. unbinds) unused scalers in hardware
2991  */
2992 static void skl_detach_scalers(struct intel_crtc *intel_crtc)
2993 {
2994         struct intel_crtc_scaler_state *scaler_state;
2995         int i;
2996
2997         scaler_state = &intel_crtc->config->scaler_state;
2998
2999         /* loop through and disable scalers that aren't in use */
3000         for (i = 0; i < intel_crtc->num_scalers; i++) {
3001                 if (!scaler_state->scalers[i].in_use)
3002                         skl_detach_scaler(intel_crtc, i);
3003         }
3004 }
3005
3006 u32 skl_plane_ctl_format(uint32_t pixel_format)
3007 {
3008         switch (pixel_format) {
3009         case DRM_FORMAT_C8:
3010                 return PLANE_CTL_FORMAT_INDEXED;
3011         case DRM_FORMAT_RGB565:
3012                 return PLANE_CTL_FORMAT_RGB_565;
3013         case DRM_FORMAT_XBGR8888:
3014                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
3015         case DRM_FORMAT_XRGB8888:
3016                 return PLANE_CTL_FORMAT_XRGB_8888;
3017         /*
3018          * XXX: For ARBG/ABGR formats we default to expecting scanout buffers
3019          * to be already pre-multiplied. We need to add a knob (or a different
3020          * DRM_FORMAT) for user-space to configure that.
3021          */
3022         case DRM_FORMAT_ABGR8888:
3023                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX |
3024                         PLANE_CTL_ALPHA_SW_PREMULTIPLY;
3025         case DRM_FORMAT_ARGB8888:
3026                 return PLANE_CTL_FORMAT_XRGB_8888 |
3027                         PLANE_CTL_ALPHA_SW_PREMULTIPLY;
3028         case DRM_FORMAT_XRGB2101010:
3029                 return PLANE_CTL_FORMAT_XRGB_2101010;
3030         case DRM_FORMAT_XBGR2101010:
3031                 return PLANE_CTL_ORDER_RGBX | PLANE_CTL_FORMAT_XRGB_2101010;
3032         case DRM_FORMAT_YUYV:
3033                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
3034         case DRM_FORMAT_YVYU:
3035                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
3036         case DRM_FORMAT_UYVY:
3037                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
3038         case DRM_FORMAT_VYUY:
3039                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
3040         default:
3041                 MISSING_CASE(pixel_format);
3042         }
3043
3044         return 0;
3045 }
3046
3047 u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
3048 {
3049         switch (fb_modifier) {
3050         case DRM_FORMAT_MOD_NONE:
3051                 break;
3052         case I915_FORMAT_MOD_X_TILED:
3053                 return PLANE_CTL_TILED_X;
3054         case I915_FORMAT_MOD_Y_TILED:
3055                 return PLANE_CTL_TILED_Y;
3056         case I915_FORMAT_MOD_Yf_TILED:
3057                 return PLANE_CTL_TILED_YF;
3058         default:
3059                 MISSING_CASE(fb_modifier);
3060         }
3061
3062         return 0;
3063 }
3064
3065 u32 skl_plane_ctl_rotation(unsigned int rotation)
3066 {
3067         switch (rotation) {
3068         case BIT(DRM_ROTATE_0):
3069                 break;
3070         /*
3071          * DRM_ROTATE_ is counter clockwise to stay compatible with Xrandr
3072          * while i915 HW rotation is clockwise, thats why this swapping.
3073          */
3074         case BIT(DRM_ROTATE_90):
3075                 return PLANE_CTL_ROTATE_270;
3076         case BIT(DRM_ROTATE_180):
3077                 return PLANE_CTL_ROTATE_180;
3078         case BIT(DRM_ROTATE_270):
3079                 return PLANE_CTL_ROTATE_90;
3080         default:
3081                 MISSING_CASE(rotation);
3082         }
3083
3084         return 0;
3085 }
3086
3087 static void skylake_update_primary_plane(struct drm_crtc *crtc,
3088                                          struct drm_framebuffer *fb,
3089                                          int x, int y)
3090 {
3091         struct drm_device *dev = crtc->dev;
3092         struct drm_i915_private *dev_priv = dev->dev_private;
3093         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3094         struct drm_plane *plane = crtc->primary;
3095         bool visible = to_intel_plane_state(plane->state)->visible;
3096         struct drm_i915_gem_object *obj;
3097         int pipe = intel_crtc->pipe;
3098         u32 plane_ctl, stride_div, stride;
3099         u32 tile_height, plane_offset, plane_size;
3100         unsigned int rotation;
3101         int x_offset, y_offset;
3102         unsigned long surf_addr;
3103         struct intel_crtc_state *crtc_state = intel_crtc->config;
3104         struct intel_plane_state *plane_state;
3105         int src_x = 0, src_y = 0, src_w = 0, src_h = 0;
3106         int dst_x = 0, dst_y = 0, dst_w = 0, dst_h = 0;
3107         int scaler_id = -1;
3108
3109         plane_state = to_intel_plane_state(plane->state);
3110
3111         if (!visible || !fb) {
3112                 I915_WRITE(PLANE_CTL(pipe, 0), 0);
3113                 I915_WRITE(PLANE_SURF(pipe, 0), 0);
3114                 POSTING_READ(PLANE_CTL(pipe, 0));
3115                 return;
3116         }
3117
3118         plane_ctl = PLANE_CTL_ENABLE |
3119                     PLANE_CTL_PIPE_GAMMA_ENABLE |
3120                     PLANE_CTL_PIPE_CSC_ENABLE;
3121
3122         plane_ctl |= skl_plane_ctl_format(fb->pixel_format);
3123         plane_ctl |= skl_plane_ctl_tiling(fb->modifier[0]);
3124         plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
3125
3126         rotation = plane->state->rotation;
3127         plane_ctl |= skl_plane_ctl_rotation(rotation);
3128
3129         obj = intel_fb_obj(fb);
3130         stride_div = intel_fb_stride_alignment(dev, fb->modifier[0],
3131                                                fb->pixel_format);
3132         surf_addr = intel_plane_obj_offset(to_intel_plane(plane), obj, 0);
3133
3134         WARN_ON(drm_rect_width(&plane_state->src) == 0);
3135
3136         scaler_id = plane_state->scaler_id;
3137         src_x = plane_state->src.x1 >> 16;
3138         src_y = plane_state->src.y1 >> 16;
3139         src_w = drm_rect_width(&plane_state->src) >> 16;
3140         src_h = drm_rect_height(&plane_state->src) >> 16;
3141         dst_x = plane_state->dst.x1;
3142         dst_y = plane_state->dst.y1;
3143         dst_w = drm_rect_width(&plane_state->dst);
3144         dst_h = drm_rect_height(&plane_state->dst);
3145
3146         WARN_ON(x != src_x || y != src_y);
3147
3148         if (intel_rotation_90_or_270(rotation)) {
3149                 /* stride = Surface height in tiles */
3150                 tile_height = intel_tile_height(dev, fb->pixel_format,
3151                                                 fb->modifier[0], 0);
3152                 stride = DIV_ROUND_UP(fb->height, tile_height);
3153                 x_offset = stride * tile_height - y - src_h;
3154                 y_offset = x;
3155                 plane_size = (src_w - 1) << 16 | (src_h - 1);
3156         } else {
3157                 stride = fb->pitches[0] / stride_div;
3158                 x_offset = x;
3159                 y_offset = y;
3160                 plane_size = (src_h - 1) << 16 | (src_w - 1);
3161         }
3162         plane_offset = y_offset << 16 | x_offset;
3163
3164         intel_crtc->adjusted_x = x_offset;
3165         intel_crtc->adjusted_y = y_offset;
3166
3167         I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl);
3168         I915_WRITE(PLANE_OFFSET(pipe, 0), plane_offset);
3169         I915_WRITE(PLANE_SIZE(pipe, 0), plane_size);
3170         I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
3171
3172         if (scaler_id >= 0) {
3173                 uint32_t ps_ctrl = 0;
3174
3175                 WARN_ON(!dst_w || !dst_h);
3176                 ps_ctrl = PS_SCALER_EN | PS_PLANE_SEL(0) |
3177                         crtc_state->scaler_state.scalers[scaler_id].mode;
3178                 I915_WRITE(SKL_PS_CTRL(pipe, scaler_id), ps_ctrl);
3179                 I915_WRITE(SKL_PS_PWR_GATE(pipe, scaler_id), 0);
3180                 I915_WRITE(SKL_PS_WIN_POS(pipe, scaler_id), (dst_x << 16) | dst_y);
3181                 I915_WRITE(SKL_PS_WIN_SZ(pipe, scaler_id), (dst_w << 16) | dst_h);
3182                 I915_WRITE(PLANE_POS(pipe, 0), 0);
3183         } else {
3184                 I915_WRITE(PLANE_POS(pipe, 0), (dst_y << 16) | dst_x);
3185         }
3186
3187         I915_WRITE(PLANE_SURF(pipe, 0), surf_addr);
3188
3189         POSTING_READ(PLANE_SURF(pipe, 0));
3190 }
3191
3192 /* Assume fb object is pinned & idle & fenced and just update base pointers */
3193 static int
3194 intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
3195                            int x, int y, enum mode_set_atomic state)
3196 {
3197         struct drm_device *dev = crtc->dev;
3198         struct drm_i915_private *dev_priv = dev->dev_private;
3199
3200         if (dev_priv->fbc.disable_fbc)
3201                 dev_priv->fbc.disable_fbc(dev_priv);
3202
3203         dev_priv->display.update_primary_plane(crtc, fb, x, y);
3204
3205         return 0;
3206 }
3207
3208 static void intel_complete_page_flips(struct drm_device *dev)
3209 {
3210         struct drm_crtc *crtc;
3211
3212         for_each_crtc(dev, crtc) {
3213                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3214                 enum plane plane = intel_crtc->plane;
3215
3216                 intel_prepare_page_flip(dev, plane);
3217                 intel_finish_page_flip_plane(dev, plane);
3218         }
3219 }
3220
3221 static void intel_update_primary_planes(struct drm_device *dev)
3222 {
3223         struct drm_crtc *crtc;
3224
3225         for_each_crtc(dev, crtc) {
3226                 struct intel_plane *plane = to_intel_plane(crtc->primary);
3227                 struct intel_plane_state *plane_state;
3228
3229                 drm_modeset_lock_crtc(crtc, &plane->base);
3230
3231                 plane_state = to_intel_plane_state(plane->base.state);
3232
3233                 if (plane_state->base.fb)
3234                         plane->commit_plane(&plane->base, plane_state);
3235
3236                 drm_modeset_unlock_crtc(crtc);
3237         }
3238 }
3239
3240 void intel_prepare_reset(struct drm_device *dev)
3241 {
3242         /* no reset support for gen2 */
3243         if (IS_GEN2(dev))
3244                 return;
3245
3246         /* reset doesn't touch the display */
3247         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
3248                 return;
3249
3250         drm_modeset_lock_all(dev);
3251         /*
3252          * Disabling the crtcs gracefully seems nicer. Also the
3253          * g33 docs say we should at least disable all the planes.
3254          */
3255         intel_display_suspend(dev);
3256 }
3257
3258 void intel_finish_reset(struct drm_device *dev)
3259 {
3260         struct drm_i915_private *dev_priv = to_i915(dev);
3261
3262         /*
3263          * Flips in the rings will be nuked by the reset,
3264          * so complete all pending flips so that user space
3265          * will get its events and not get stuck.
3266          */
3267         intel_complete_page_flips(dev);
3268
3269         /* no reset support for gen2 */
3270         if (IS_GEN2(dev))
3271                 return;
3272
3273         /* reset doesn't touch the display */
3274         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev)) {
3275                 /*
3276                  * Flips in the rings have been nuked by the reset,
3277                  * so update the base address of all primary
3278                  * planes to the the last fb to make sure we're
3279                  * showing the correct fb after a reset.
3280                  *
3281                  * FIXME: Atomic will make this obsolete since we won't schedule
3282                  * CS-based flips (which might get lost in gpu resets) any more.
3283                  */
3284                 intel_update_primary_planes(dev);
3285                 return;
3286         }
3287
3288         /*
3289          * The display has been reset as well,
3290          * so need a full re-initialization.
3291          */
3292         intel_runtime_pm_disable_interrupts(dev_priv);
3293         intel_runtime_pm_enable_interrupts(dev_priv);
3294
3295         intel_modeset_init_hw(dev);
3296
3297         spin_lock_irq(&dev_priv->irq_lock);
3298         if (dev_priv->display.hpd_irq_setup)
3299                 dev_priv->display.hpd_irq_setup(dev);
3300         spin_unlock_irq(&dev_priv->irq_lock);
3301
3302         intel_display_resume(dev);
3303
3304         intel_hpd_init(dev_priv);
3305
3306         drm_modeset_unlock_all(dev);
3307 }
3308
3309 static void
3310 intel_finish_fb(struct drm_framebuffer *old_fb)
3311 {
3312         struct drm_i915_gem_object *obj = intel_fb_obj(old_fb);
3313         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3314         bool was_interruptible = dev_priv->mm.interruptible;
3315         int ret;
3316
3317         /* Big Hammer, we also need to ensure that any pending
3318          * MI_WAIT_FOR_EVENT inside a user batch buffer on the
3319          * current scanout is retired before unpinning the old
3320          * framebuffer. Note that we rely on userspace rendering
3321          * into the buffer attached to the pipe they are waiting
3322          * on. If not, userspace generates a GPU hang with IPEHR
3323          * point to the MI_WAIT_FOR_EVENT.
3324          *
3325          * This should only fail upon a hung GPU, in which case we
3326          * can safely continue.
3327          */
3328         dev_priv->mm.interruptible = false;
3329         ret = i915_gem_object_wait_rendering(obj, true);
3330         dev_priv->mm.interruptible = was_interruptible;
3331
3332         WARN_ON(ret);
3333 }
3334
3335 static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
3336 {
3337         struct drm_device *dev = crtc->dev;
3338         struct drm_i915_private *dev_priv = dev->dev_private;
3339         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3340         bool pending;
3341
3342         if (i915_reset_in_progress(&dev_priv->gpu_error) ||
3343             intel_crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
3344                 return false;
3345
3346         spin_lock_irq(&dev->event_lock);
3347         pending = to_intel_crtc(crtc)->unpin_work != NULL;
3348         spin_unlock_irq(&dev->event_lock);
3349
3350         return pending;
3351 }
3352
3353 static void intel_update_pipe_config(struct intel_crtc *crtc,
3354                                      struct intel_crtc_state *old_crtc_state)
3355 {
3356         struct drm_device *dev = crtc->base.dev;
3357         struct drm_i915_private *dev_priv = dev->dev_private;
3358         struct intel_crtc_state *pipe_config =
3359                 to_intel_crtc_state(crtc->base.state);
3360
3361         /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
3362         crtc->base.mode = crtc->base.state->mode;
3363
3364         DRM_DEBUG_KMS("Updating pipe size %ix%i -> %ix%i\n",
3365                       old_crtc_state->pipe_src_w, old_crtc_state->pipe_src_h,
3366                       pipe_config->pipe_src_w, pipe_config->pipe_src_h);
3367
3368         if (HAS_DDI(dev))
3369                 intel_set_pipe_csc(&crtc->base);
3370
3371         /*
3372          * Update pipe size and adjust fitter if needed: the reason for this is
3373          * that in compute_mode_changes we check the native mode (not the pfit
3374          * mode) to see if we can flip rather than do a full mode set. In the
3375          * fastboot case, we'll flip, but if we don't update the pipesrc and
3376          * pfit state, we'll end up with a big fb scanned out into the wrong
3377          * sized surface.
3378          */
3379
3380         I915_WRITE(PIPESRC(crtc->pipe),
3381                    ((pipe_config->pipe_src_w - 1) << 16) |
3382                    (pipe_config->pipe_src_h - 1));
3383
3384         /* on skylake this is done by detaching scalers */
3385         if (INTEL_INFO(dev)->gen >= 9) {
3386                 skl_detach_scalers(crtc);
3387
3388                 if (pipe_config->pch_pfit.enabled)
3389                         skylake_pfit_enable(crtc);
3390         } else if (HAS_PCH_SPLIT(dev)) {
3391                 if (pipe_config->pch_pfit.enabled)
3392                         ironlake_pfit_enable(crtc);
3393                 else if (old_crtc_state->pch_pfit.enabled)
3394                         ironlake_pfit_disable(crtc, true);
3395         }
3396 }
3397
3398 static void intel_fdi_normal_train(struct drm_crtc *crtc)
3399 {
3400         struct drm_device *dev = crtc->dev;
3401         struct drm_i915_private *dev_priv = dev->dev_private;
3402         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3403         int pipe = intel_crtc->pipe;
3404         u32 reg, temp;
3405
3406         /* enable normal train */
3407         reg = FDI_TX_CTL(pipe);
3408         temp = I915_READ(reg);
3409         if (IS_IVYBRIDGE(dev)) {
3410                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3411                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
3412         } else {
3413                 temp &= ~FDI_LINK_TRAIN_NONE;
3414                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
3415         }
3416         I915_WRITE(reg, temp);
3417
3418         reg = FDI_RX_CTL(pipe);
3419         temp = I915_READ(reg);
3420         if (HAS_PCH_CPT(dev)) {
3421                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3422                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
3423         } else {
3424                 temp &= ~FDI_LINK_TRAIN_NONE;
3425                 temp |= FDI_LINK_TRAIN_NONE;
3426         }
3427         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
3428
3429         /* wait one idle pattern time */
3430         POSTING_READ(reg);
3431         udelay(1000);
3432
3433         /* IVB wants error correction enabled */
3434         if (IS_IVYBRIDGE(dev))
3435                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
3436                            FDI_FE_ERRC_ENABLE);
3437 }
3438
3439 /* The FDI link training functions for ILK/Ibexpeak. */
3440 static void ironlake_fdi_link_train(struct drm_crtc *crtc)
3441 {
3442         struct drm_device *dev = crtc->dev;
3443         struct drm_i915_private *dev_priv = dev->dev_private;
3444         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3445         int pipe = intel_crtc->pipe;
3446         u32 reg, temp, tries;
3447
3448         /* FDI needs bits from pipe first */
3449         assert_pipe_enabled(dev_priv, pipe);
3450
3451         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3452            for train result */
3453         reg = FDI_RX_IMR(pipe);
3454         temp = I915_READ(reg);
3455         temp &= ~FDI_RX_SYMBOL_LOCK;
3456         temp &= ~FDI_RX_BIT_LOCK;
3457         I915_WRITE(reg, temp);
3458         I915_READ(reg);
3459         udelay(150);
3460
3461         /* enable CPU FDI TX and PCH FDI RX */
3462         reg = FDI_TX_CTL(pipe);
3463         temp = I915_READ(reg);
3464         temp &= ~FDI_DP_PORT_WIDTH_MASK;
3465         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3466         temp &= ~FDI_LINK_TRAIN_NONE;
3467         temp |= FDI_LINK_TRAIN_PATTERN_1;
3468         I915_WRITE(reg, temp | FDI_TX_ENABLE);
3469
3470         reg = FDI_RX_CTL(pipe);
3471         temp = I915_READ(reg);
3472         temp &= ~FDI_LINK_TRAIN_NONE;
3473         temp |= FDI_LINK_TRAIN_PATTERN_1;
3474         I915_WRITE(reg, temp | FDI_RX_ENABLE);
3475
3476         POSTING_READ(reg);
3477         udelay(150);
3478
3479         /* Ironlake workaround, enable clock pointer after FDI enable*/
3480         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
3481         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
3482                    FDI_RX_PHASE_SYNC_POINTER_EN);
3483
3484         reg = FDI_RX_IIR(pipe);
3485         for (tries = 0; tries < 5; tries++) {
3486                 temp = I915_READ(reg);
3487                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3488
3489                 if ((temp & FDI_RX_BIT_LOCK)) {
3490                         DRM_DEBUG_KMS("FDI train 1 done.\n");
3491                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3492                         break;
3493                 }
3494         }
3495         if (tries == 5)
3496                 DRM_ERROR("FDI train 1 fail!\n");
3497
3498         /* Train 2 */
3499         reg = FDI_TX_CTL(pipe);
3500         temp = I915_READ(reg);
3501         temp &= ~FDI_LINK_TRAIN_NONE;
3502         temp |= FDI_LINK_TRAIN_PATTERN_2;
3503         I915_WRITE(reg, temp);
3504
3505         reg = FDI_RX_CTL(pipe);
3506         temp = I915_READ(reg);
3507         temp &= ~FDI_LINK_TRAIN_NONE;
3508         temp |= FDI_LINK_TRAIN_PATTERN_2;
3509         I915_WRITE(reg, temp);
3510
3511         POSTING_READ(reg);
3512         udelay(150);
3513
3514         reg = FDI_RX_IIR(pipe);
3515         for (tries = 0; tries < 5; tries++) {
3516                 temp = I915_READ(reg);
3517                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3518
3519                 if (temp & FDI_RX_SYMBOL_LOCK) {
3520                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3521                         DRM_DEBUG_KMS("FDI train 2 done.\n");
3522                         break;
3523                 }
3524         }
3525         if (tries == 5)
3526                 DRM_ERROR("FDI train 2 fail!\n");
3527
3528         DRM_DEBUG_KMS("FDI train done\n");
3529
3530 }
3531
3532 static const int snb_b_fdi_train_param[] = {
3533         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
3534         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
3535         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
3536         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
3537 };
3538
3539 /* The FDI link training functions for SNB/Cougarpoint. */
3540 static void gen6_fdi_link_train(struct drm_crtc *crtc)
3541 {
3542         struct drm_device *dev = crtc->dev;
3543         struct drm_i915_private *dev_priv = dev->dev_private;
3544         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3545         int pipe = intel_crtc->pipe;
3546         u32 reg, temp, i, retry;
3547
3548         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3549            for train result */
3550         reg = FDI_RX_IMR(pipe);
3551         temp = I915_READ(reg);
3552         temp &= ~FDI_RX_SYMBOL_LOCK;
3553         temp &= ~FDI_RX_BIT_LOCK;
3554         I915_WRITE(reg, temp);
3555
3556         POSTING_READ(reg);
3557         udelay(150);
3558
3559         /* enable CPU FDI TX and PCH FDI RX */
3560         reg = FDI_TX_CTL(pipe);
3561         temp = I915_READ(reg);
3562         temp &= ~FDI_DP_PORT_WIDTH_MASK;
3563         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3564         temp &= ~FDI_LINK_TRAIN_NONE;
3565         temp |= FDI_LINK_TRAIN_PATTERN_1;
3566         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3567         /* SNB-B */
3568         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
3569         I915_WRITE(reg, temp | FDI_TX_ENABLE);
3570
3571         I915_WRITE(FDI_RX_MISC(pipe),
3572                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
3573
3574         reg = FDI_RX_CTL(pipe);
3575         temp = I915_READ(reg);
3576         if (HAS_PCH_CPT(dev)) {
3577                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3578                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3579         } else {
3580                 temp &= ~FDI_LINK_TRAIN_NONE;
3581                 temp |= FDI_LINK_TRAIN_PATTERN_1;
3582         }
3583         I915_WRITE(reg, temp | FDI_RX_ENABLE);
3584
3585         POSTING_READ(reg);
3586         udelay(150);
3587
3588         for (i = 0; i < 4; i++) {
3589                 reg = FDI_TX_CTL(pipe);
3590                 temp = I915_READ(reg);
3591                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3592                 temp |= snb_b_fdi_train_param[i];
3593                 I915_WRITE(reg, temp);
3594
3595                 POSTING_READ(reg);
3596                 udelay(500);
3597
3598                 for (retry = 0; retry < 5; retry++) {
3599                         reg = FDI_RX_IIR(pipe);
3600                         temp = I915_READ(reg);
3601                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3602                         if (temp & FDI_RX_BIT_LOCK) {
3603                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3604                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
3605                                 break;
3606                         }
3607                         udelay(50);
3608                 }
3609                 if (retry < 5)
3610                         break;
3611         }
3612         if (i == 4)
3613                 DRM_ERROR("FDI train 1 fail!\n");
3614
3615         /* Train 2 */
3616         reg = FDI_TX_CTL(pipe);
3617         temp = I915_READ(reg);
3618         temp &= ~FDI_LINK_TRAIN_NONE;
3619         temp |= FDI_LINK_TRAIN_PATTERN_2;
3620         if (IS_GEN6(dev)) {
3621                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3622                 /* SNB-B */
3623                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
3624         }
3625         I915_WRITE(reg, temp);
3626
3627         reg = FDI_RX_CTL(pipe);
3628         temp = I915_READ(reg);
3629         if (HAS_PCH_CPT(dev)) {
3630                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3631                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
3632         } else {
3633                 temp &= ~FDI_LINK_TRAIN_NONE;
3634                 temp |= FDI_LINK_TRAIN_PATTERN_2;
3635         }
3636         I915_WRITE(reg, temp);
3637
3638         POSTING_READ(reg);
3639         udelay(150);
3640
3641         for (i = 0; i < 4; i++) {
3642                 reg = FDI_TX_CTL(pipe);
3643                 temp = I915_READ(reg);
3644                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3645                 temp |= snb_b_fdi_train_param[i];
3646                 I915_WRITE(reg, temp);
3647
3648                 POSTING_READ(reg);
3649                 udelay(500);
3650
3651                 for (retry = 0; retry < 5; retry++) {
3652                         reg = FDI_RX_IIR(pipe);
3653                         temp = I915_READ(reg);
3654                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3655                         if (temp & FDI_RX_SYMBOL_LOCK) {
3656                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3657                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
3658                                 break;
3659                         }
3660                         udelay(50);
3661                 }
3662                 if (retry < 5)
3663                         break;
3664         }
3665         if (i == 4)
3666                 DRM_ERROR("FDI train 2 fail!\n");
3667
3668         DRM_DEBUG_KMS("FDI train done.\n");
3669 }
3670
3671 /* Manual link training for Ivy Bridge A0 parts */
3672 static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
3673 {
3674         struct drm_device *dev = crtc->dev;
3675         struct drm_i915_private *dev_priv = dev->dev_private;
3676         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3677         int pipe = intel_crtc->pipe;
3678         u32 reg, temp, i, j;
3679
3680         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3681            for train result */
3682         reg = FDI_RX_IMR(pipe);
3683         temp = I915_READ(reg);
3684         temp &= ~FDI_RX_SYMBOL_LOCK;
3685         temp &= ~FDI_RX_BIT_LOCK;
3686         I915_WRITE(reg, temp);
3687
3688         POSTING_READ(reg);
3689         udelay(150);
3690
3691         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
3692                       I915_READ(FDI_RX_IIR(pipe)));
3693
3694         /* Try each vswing and preemphasis setting twice before moving on */
3695         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
3696                 /* disable first in case we need to retry */
3697                 reg = FDI_TX_CTL(pipe);
3698                 temp = I915_READ(reg);
3699                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
3700                 temp &= ~FDI_TX_ENABLE;
3701                 I915_WRITE(reg, temp);
3702
3703                 reg = FDI_RX_CTL(pipe);
3704                 temp = I915_READ(reg);
3705                 temp &= ~FDI_LINK_TRAIN_AUTO;
3706                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3707                 temp &= ~FDI_RX_ENABLE;
3708                 I915_WRITE(reg, temp);
3709
3710                 /* enable CPU FDI TX and PCH FDI RX */
3711                 reg = FDI_TX_CTL(pipe);
3712                 temp = I915_READ(reg);
3713                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
3714                 temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3715                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
3716                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3717                 temp |= snb_b_fdi_train_param[j/2];
3718                 temp |= FDI_COMPOSITE_SYNC;
3719                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
3720
3721                 I915_WRITE(FDI_RX_MISC(pipe),
3722                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
3723
3724                 reg = FDI_RX_CTL(pipe);
3725                 temp = I915_READ(reg);
3726                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3727                 temp |= FDI_COMPOSITE_SYNC;
3728                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
3729
3730                 POSTING_READ(reg);
3731                 udelay(1); /* should be 0.5us */
3732
3733                 for (i = 0; i < 4; i++) {
3734                         reg = FDI_RX_IIR(pipe);
3735                         temp = I915_READ(reg);
3736                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3737
3738                         if (temp & FDI_RX_BIT_LOCK ||
3739                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
3740                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3741                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
3742                                               i);
3743                                 break;
3744                         }
3745                         udelay(1); /* should be 0.5us */
3746                 }
3747                 if (i == 4) {
3748                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
3749                         continue;
3750                 }
3751
3752                 /* Train 2 */
3753                 reg = FDI_TX_CTL(pipe);
3754                 temp = I915_READ(reg);
3755                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3756                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
3757                 I915_WRITE(reg, temp);
3758
3759                 reg = FDI_RX_CTL(pipe);
3760                 temp = I915_READ(reg);
3761                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3762                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
3763                 I915_WRITE(reg, temp);
3764
3765                 POSTING_READ(reg);
3766                 udelay(2); /* should be 1.5us */
3767
3768                 for (i = 0; i < 4; i++) {
3769                         reg = FDI_RX_IIR(pipe);
3770                         temp = I915_READ(reg);
3771                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3772
3773                         if (temp & FDI_RX_SYMBOL_LOCK ||
3774                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
3775                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3776                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
3777                                               i);
3778                                 goto train_done;
3779                         }
3780                         udelay(2); /* should be 1.5us */
3781                 }
3782                 if (i == 4)
3783                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
3784         }
3785
3786 train_done:
3787         DRM_DEBUG_KMS("FDI train done.\n");
3788 }
3789
3790 static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc)
3791 {
3792         struct drm_device *dev = intel_crtc->base.dev;
3793         struct drm_i915_private *dev_priv = dev->dev_private;
3794         int pipe = intel_crtc->pipe;
3795         u32 reg, temp;
3796
3797
3798         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
3799         reg = FDI_RX_CTL(pipe);
3800         temp = I915_READ(reg);
3801         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
3802         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3803         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3804         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
3805
3806         POSTING_READ(reg);
3807         udelay(200);
3808
3809         /* Switch from Rawclk to PCDclk */
3810         temp = I915_READ(reg);
3811         I915_WRITE(reg, temp | FDI_PCDCLK);
3812
3813         POSTING_READ(reg);
3814         udelay(200);
3815
3816         /* Enable CPU FDI TX PLL, always on for Ironlake */
3817         reg = FDI_TX_CTL(pipe);
3818         temp = I915_READ(reg);
3819         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
3820                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
3821
3822                 POSTING_READ(reg);
3823                 udelay(100);
3824         }
3825 }
3826
3827 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
3828 {
3829         struct drm_device *dev = intel_crtc->base.dev;
3830         struct drm_i915_private *dev_priv = dev->dev_private;
3831         int pipe = intel_crtc->pipe;
3832         u32 reg, temp;
3833
3834         /* Switch from PCDclk to Rawclk */
3835         reg = FDI_RX_CTL(pipe);
3836         temp = I915_READ(reg);
3837         I915_WRITE(reg, temp & ~FDI_PCDCLK);
3838
3839         /* Disable CPU FDI TX PLL */
3840         reg = FDI_TX_CTL(pipe);
3841         temp = I915_READ(reg);
3842         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
3843
3844         POSTING_READ(reg);
3845         udelay(100);
3846
3847         reg = FDI_RX_CTL(pipe);
3848         temp = I915_READ(reg);
3849         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
3850
3851         /* Wait for the clocks to turn off. */
3852         POSTING_READ(reg);
3853         udelay(100);
3854 }
3855
3856 static void ironlake_fdi_disable(struct drm_crtc *crtc)
3857 {
3858         struct drm_device *dev = crtc->dev;
3859         struct drm_i915_private *dev_priv = dev->dev_private;
3860         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3861         int pipe = intel_crtc->pipe;
3862         u32 reg, temp;
3863
3864         /* disable CPU FDI tx and PCH FDI rx */
3865         reg = FDI_TX_CTL(pipe);
3866         temp = I915_READ(reg);
3867         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
3868         POSTING_READ(reg);
3869
3870         reg = FDI_RX_CTL(pipe);
3871         temp = I915_READ(reg);
3872         temp &= ~(0x7 << 16);
3873         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3874         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
3875
3876         POSTING_READ(reg);
3877         udelay(100);
3878
3879         /* Ironlake workaround, disable clock pointer after downing FDI */
3880         if (HAS_PCH_IBX(dev))
3881                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
3882
3883         /* still set train pattern 1 */
3884         reg = FDI_TX_CTL(pipe);
3885         temp = I915_READ(reg);
3886         temp &= ~FDI_LINK_TRAIN_NONE;
3887         temp |= FDI_LINK_TRAIN_PATTERN_1;
3888         I915_WRITE(reg, temp);
3889
3890         reg = FDI_RX_CTL(pipe);
3891         temp = I915_READ(reg);
3892         if (HAS_PCH_CPT(dev)) {
3893                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3894                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3895         } else {
3896                 temp &= ~FDI_LINK_TRAIN_NONE;
3897                 temp |= FDI_LINK_TRAIN_PATTERN_1;
3898         }
3899         /* BPC in FDI rx is consistent with that in PIPECONF */
3900         temp &= ~(0x07 << 16);
3901         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3902         I915_WRITE(reg, temp);
3903
3904         POSTING_READ(reg);
3905         udelay(100);
3906 }
3907
3908 bool intel_has_pending_fb_unpin(struct drm_device *dev)
3909 {
3910         struct intel_crtc *crtc;
3911
3912         /* Note that we don't need to be called with mode_config.lock here
3913          * as our list of CRTC objects is static for the lifetime of the
3914          * device and so cannot disappear as we iterate. Similarly, we can
3915          * happily treat the predicates as racy, atomic checks as userspace
3916          * cannot claim and pin a new fb without at least acquring the
3917          * struct_mutex and so serialising with us.
3918          */
3919         for_each_intel_crtc(dev, crtc) {
3920                 if (atomic_read(&crtc->unpin_work_count) == 0)
3921                         continue;
3922
3923                 if (crtc->unpin_work)
3924                         intel_wait_for_vblank(dev, crtc->pipe);
3925
3926                 return true;
3927         }
3928
3929         return false;
3930 }
3931
3932 static void page_flip_completed(struct intel_crtc *intel_crtc)
3933 {
3934         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3935         struct intel_unpin_work *work = intel_crtc->unpin_work;
3936
3937         /* ensure that the unpin work is consistent wrt ->pending. */
3938         smp_rmb();
3939         intel_crtc->unpin_work = NULL;
3940
3941         if (work->event)
3942                 drm_send_vblank_event(intel_crtc->base.dev,
3943                                       intel_crtc->pipe,
3944                                       work->event);
3945
3946         drm_crtc_vblank_put(&intel_crtc->base);
3947
3948         wake_up_all(&dev_priv->pending_flip_queue);
3949         queue_work(dev_priv->wq, &work->work);
3950
3951         trace_i915_flip_complete(intel_crtc->plane,
3952                                  work->pending_flip_obj);
3953 }
3954
3955 void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
3956 {
3957         struct drm_device *dev = crtc->dev;
3958         struct drm_i915_private *dev_priv = dev->dev_private;
3959
3960         WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue));
3961         if (WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue,
3962                                        !intel_crtc_has_pending_flip(crtc),
3963                                        60*HZ) == 0)) {
3964                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3965
3966                 spin_lock_irq(&dev->event_lock);
3967                 if (intel_crtc->unpin_work) {
3968                         WARN_ONCE(1, "Removing stuck page flip\n");
3969                         page_flip_completed(intel_crtc);
3970                 }
3971                 spin_unlock_irq(&dev->event_lock);
3972         }
3973
3974         if (crtc->primary->fb) {
3975                 mutex_lock(&dev->struct_mutex);
3976                 intel_finish_fb(crtc->primary->fb);
3977                 mutex_unlock(&dev->struct_mutex);
3978         }
3979 }
3980
3981 /* Program iCLKIP clock to the desired frequency */
3982 static void lpt_program_iclkip(struct drm_crtc *crtc)
3983 {
3984         struct drm_device *dev = crtc->dev;
3985         struct drm_i915_private *dev_priv = dev->dev_private;
3986         int clock = to_intel_crtc(crtc)->config->base.adjusted_mode.crtc_clock;
3987         u32 divsel, phaseinc, auxdiv, phasedir = 0;
3988         u32 temp;
3989
3990         mutex_lock(&dev_priv->sb_lock);
3991
3992         /* It is necessary to ungate the pixclk gate prior to programming
3993          * the divisors, and gate it back when it is done.
3994          */
3995         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
3996
3997         /* Disable SSCCTL */
3998         intel_sbi_write(dev_priv, SBI_SSCCTL6,
3999                         intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK) |
4000                                 SBI_SSCCTL_DISABLE,
4001                         SBI_ICLK);
4002
4003         /* 20MHz is a corner case which is out of range for the 7-bit divisor */
4004         if (clock == 20000) {
4005                 auxdiv = 1;
4006                 divsel = 0x41;
4007                 phaseinc = 0x20;
4008         } else {
4009                 /* The iCLK virtual clock root frequency is in MHz,
4010                  * but the adjusted_mode->crtc_clock in in KHz. To get the
4011                  * divisors, it is necessary to divide one by another, so we
4012                  * convert the virtual clock precision to KHz here for higher
4013                  * precision.
4014                  */
4015                 u32 iclk_virtual_root_freq = 172800 * 1000;
4016                 u32 iclk_pi_range = 64;
4017                 u32 desired_divisor, msb_divisor_value, pi_value;
4018
4019                 desired_divisor = (iclk_virtual_root_freq / clock);
4020                 msb_divisor_value = desired_divisor / iclk_pi_range;
4021                 pi_value = desired_divisor % iclk_pi_range;
4022
4023                 auxdiv = 0;
4024                 divsel = msb_divisor_value - 2;
4025                 phaseinc = pi_value;
4026         }
4027
4028         /* This should not happen with any sane values */
4029         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
4030                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
4031         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
4032                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
4033
4034         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
4035                         clock,
4036                         auxdiv,
4037                         divsel,
4038                         phasedir,
4039                         phaseinc);
4040
4041         /* Program SSCDIVINTPHASE6 */
4042         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
4043         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
4044         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
4045         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
4046         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
4047         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
4048         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
4049         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
4050
4051         /* Program SSCAUXDIV */
4052         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
4053         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
4054         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
4055         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
4056
4057         /* Enable modulator and associated divider */
4058         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4059         temp &= ~SBI_SSCCTL_DISABLE;
4060         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4061
4062         /* Wait for initialization time */
4063         udelay(24);
4064
4065         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
4066
4067         mutex_unlock(&dev_priv->sb_lock);
4068 }
4069
4070 static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc,
4071                                                 enum pipe pch_transcoder)
4072 {
4073         struct drm_device *dev = crtc->base.dev;
4074         struct drm_i915_private *dev_priv = dev->dev_private;
4075         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
4076
4077         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
4078                    I915_READ(HTOTAL(cpu_transcoder)));
4079         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
4080                    I915_READ(HBLANK(cpu_transcoder)));
4081         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
4082                    I915_READ(HSYNC(cpu_transcoder)));
4083
4084         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
4085                    I915_READ(VTOTAL(cpu_transcoder)));
4086         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
4087                    I915_READ(VBLANK(cpu_transcoder)));
4088         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
4089                    I915_READ(VSYNC(cpu_transcoder)));
4090         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
4091                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
4092 }
4093
4094 static void cpt_set_fdi_bc_bifurcation(struct drm_device *dev, bool enable)
4095 {
4096         struct drm_i915_private *dev_priv = dev->dev_private;
4097         uint32_t temp;
4098
4099         temp = I915_READ(SOUTH_CHICKEN1);
4100         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
4101                 return;
4102
4103         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
4104         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
4105
4106         temp &= ~FDI_BC_BIFURCATION_SELECT;
4107         if (enable)
4108                 temp |= FDI_BC_BIFURCATION_SELECT;
4109
4110         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
4111         I915_WRITE(SOUTH_CHICKEN1, temp);
4112         POSTING_READ(SOUTH_CHICKEN1);
4113 }
4114
4115 static void ivybridge_update_fdi_bc_bifurcation(struct intel_crtc *intel_crtc)
4116 {
4117         struct drm_device *dev = intel_crtc->base.dev;
4118
4119         switch (intel_crtc->pipe) {
4120         case PIPE_A:
4121                 break;
4122         case PIPE_B:
4123                 if (intel_crtc->config->fdi_lanes > 2)
4124                         cpt_set_fdi_bc_bifurcation(dev, false);
4125                 else
4126                         cpt_set_fdi_bc_bifurcation(dev, true);
4127
4128                 break;
4129         case PIPE_C:
4130                 cpt_set_fdi_bc_bifurcation(dev, true);
4131
4132                 break;
4133         default:
4134                 BUG();
4135         }
4136 }
4137
4138 /*
4139  * Enable PCH resources required for PCH ports:
4140  *   - PCH PLLs
4141  *   - FDI training & RX/TX
4142  *   - update transcoder timings
4143  *   - DP transcoding bits
4144  *   - transcoder
4145  */
4146 static void ironlake_pch_enable(struct drm_crtc *crtc)
4147 {
4148         struct drm_device *dev = crtc->dev;
4149         struct drm_i915_private *dev_priv = dev->dev_private;
4150         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4151         int pipe = intel_crtc->pipe;
4152         u32 reg, temp;
4153
4154         assert_pch_transcoder_disabled(dev_priv, pipe);
4155
4156         if (IS_IVYBRIDGE(dev))
4157                 ivybridge_update_fdi_bc_bifurcation(intel_crtc);
4158
4159         /* Write the TU size bits before fdi link training, so that error
4160          * detection works. */
4161         I915_WRITE(FDI_RX_TUSIZE1(pipe),
4162                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
4163
4164         /* For PCH output, training FDI link */
4165         dev_priv->display.fdi_link_train(crtc);
4166
4167         /* We need to program the right clock selection before writing the pixel
4168          * mutliplier into the DPLL. */
4169         if (HAS_PCH_CPT(dev)) {
4170                 u32 sel;
4171
4172                 temp = I915_READ(PCH_DPLL_SEL);
4173                 temp |= TRANS_DPLL_ENABLE(pipe);
4174                 sel = TRANS_DPLLB_SEL(pipe);
4175                 if (intel_crtc->config->shared_dpll == DPLL_ID_PCH_PLL_B)
4176                         temp |= sel;
4177                 else
4178                         temp &= ~sel;
4179                 I915_WRITE(PCH_DPLL_SEL, temp);
4180         }
4181
4182         /* XXX: pch pll's can be enabled any time before we enable the PCH
4183          * transcoder, and we actually should do this to not upset any PCH
4184          * transcoder that already use the clock when we share it.
4185          *
4186          * Note that enable_shared_dpll tries to do the right thing, but
4187          * get_shared_dpll unconditionally resets the pll - we need that to have
4188          * the right LVDS enable sequence. */
4189         intel_enable_shared_dpll(intel_crtc);
4190
4191         /* set transcoder timing, panel must allow it */
4192         assert_panel_unlocked(dev_priv, pipe);
4193         ironlake_pch_transcoder_set_timings(intel_crtc, pipe);
4194
4195         intel_fdi_normal_train(crtc);
4196
4197         /* For PCH DP, enable TRANS_DP_CTL */
4198         if (HAS_PCH_CPT(dev) && intel_crtc->config->has_dp_encoder) {
4199                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
4200                 reg = TRANS_DP_CTL(pipe);
4201                 temp = I915_READ(reg);
4202                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
4203                           TRANS_DP_SYNC_MASK |
4204                           TRANS_DP_BPC_MASK);
4205                 temp |= TRANS_DP_OUTPUT_ENABLE;
4206                 temp |= bpc << 9; /* same format but at 11:9 */
4207
4208                 if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
4209                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
4210                 if (crtc->mode.flags & DRM_MODE_FLAG_PVSYNC)
4211                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
4212
4213                 switch (intel_trans_dp_port_sel(crtc)) {
4214                 case PCH_DP_B:
4215                         temp |= TRANS_DP_PORT_SEL_B;
4216                         break;
4217                 case PCH_DP_C:
4218                         temp |= TRANS_DP_PORT_SEL_C;
4219                         break;
4220                 case PCH_DP_D:
4221                         temp |= TRANS_DP_PORT_SEL_D;
4222                         break;
4223                 default:
4224                         BUG();
4225                 }
4226
4227                 I915_WRITE(reg, temp);
4228         }
4229
4230         ironlake_enable_pch_transcoder(dev_priv, pipe);
4231 }
4232
4233 static void lpt_pch_enable(struct drm_crtc *crtc)
4234 {
4235         struct drm_device *dev = crtc->dev;
4236         struct drm_i915_private *dev_priv = dev->dev_private;
4237         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4238         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
4239
4240         assert_pch_transcoder_disabled(dev_priv, TRANSCODER_A);
4241
4242         lpt_program_iclkip(crtc);
4243
4244         /* Set transcoder timing. */
4245         ironlake_pch_transcoder_set_timings(intel_crtc, PIPE_A);
4246
4247         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
4248 }
4249
4250 struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc,
4251                                                 struct intel_crtc_state *crtc_state)
4252 {
4253         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
4254         struct intel_shared_dpll *pll;
4255         struct intel_shared_dpll_config *shared_dpll;
4256         enum intel_dpll_id i;
4257         int max = dev_priv->num_shared_dpll;
4258
4259         shared_dpll = intel_atomic_get_shared_dpll_state(crtc_state->base.state);
4260
4261         if (HAS_PCH_IBX(dev_priv->dev)) {
4262                 /* Ironlake PCH has a fixed PLL->PCH pipe mapping. */
4263                 i = (enum intel_dpll_id) crtc->pipe;
4264                 pll = &dev_priv->shared_dplls[i];
4265
4266                 DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n",
4267                               crtc->base.base.id, pll->name);
4268
4269                 WARN_ON(shared_dpll[i].crtc_mask);
4270
4271                 goto found;
4272         }
4273
4274         if (IS_BROXTON(dev_priv->dev)) {
4275                 /* PLL is attached to port in bxt */
4276                 struct intel_encoder *encoder;
4277                 struct intel_digital_port *intel_dig_port;
4278
4279                 encoder = intel_ddi_get_crtc_new_encoder(crtc_state);
4280                 if (WARN_ON(!encoder))
4281                         return NULL;
4282
4283                 intel_dig_port = enc_to_dig_port(&encoder->base);
4284                 /* 1:1 mapping between ports and PLLs */
4285                 i = (enum intel_dpll_id)intel_dig_port->port;
4286                 pll = &dev_priv->shared_dplls[i];
4287                 DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n",
4288                         crtc->base.base.id, pll->name);
4289                 WARN_ON(shared_dpll[i].crtc_mask);
4290
4291                 goto found;
4292         } else if (INTEL_INFO(dev_priv)->gen < 9 && HAS_DDI(dev_priv))
4293                 /* Do not consider SPLL */
4294                 max = 2;
4295
4296         for (i = 0; i < max; i++) {
4297                 pll = &dev_priv->shared_dplls[i];
4298
4299                 /* Only want to check enabled timings first */
4300                 if (shared_dpll[i].crtc_mask == 0)
4301                         continue;
4302
4303                 if (memcmp(&crtc_state->dpll_hw_state,
4304                            &shared_dpll[i].hw_state,
4305                            sizeof(crtc_state->dpll_hw_state)) == 0) {
4306                         DRM_DEBUG_KMS("CRTC:%d sharing existing %s (crtc mask 0x%08x, ative %d)\n",
4307                                       crtc->base.base.id, pll->name,
4308                                       shared_dpll[i].crtc_mask,
4309                                       pll->active);
4310                         goto found;
4311                 }
4312         }
4313
4314         /* Ok no matching timings, maybe there's a free one? */
4315         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
4316                 pll = &dev_priv->shared_dplls[i];
4317                 if (shared_dpll[i].crtc_mask == 0) {
4318                         DRM_DEBUG_KMS("CRTC:%d allocated %s\n",
4319                                       crtc->base.base.id, pll->name);
4320                         goto found;
4321                 }
4322         }
4323
4324         return NULL;
4325
4326 found:
4327         if (shared_dpll[i].crtc_mask == 0)
4328                 shared_dpll[i].hw_state =
4329                         crtc_state->dpll_hw_state;
4330
4331         crtc_state->shared_dpll = i;
4332         DRM_DEBUG_DRIVER("using %s for pipe %c\n", pll->name,
4333                          pipe_name(crtc->pipe));
4334
4335         shared_dpll[i].crtc_mask |= 1 << crtc->pipe;
4336
4337         return pll;
4338 }
4339
4340 static void intel_shared_dpll_commit(struct drm_atomic_state *state)
4341 {
4342         struct drm_i915_private *dev_priv = to_i915(state->dev);
4343         struct intel_shared_dpll_config *shared_dpll;
4344         struct intel_shared_dpll *pll;
4345         enum intel_dpll_id i;
4346
4347         if (!to_intel_atomic_state(state)->dpll_set)
4348                 return;
4349
4350         shared_dpll = to_intel_atomic_state(state)->shared_dpll;
4351         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
4352                 pll = &dev_priv->shared_dplls[i];
4353                 pll->config = shared_dpll[i];
4354         }
4355 }
4356
4357 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
4358 {
4359         struct drm_i915_private *dev_priv = dev->dev_private;
4360         int dslreg = PIPEDSL(pipe);
4361         u32 temp;
4362
4363         temp = I915_READ(dslreg);
4364         udelay(500);
4365         if (wait_for(I915_READ(dslreg) != temp, 5)) {
4366                 if (wait_for(I915_READ(dslreg) != temp, 5))
4367                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
4368         }
4369 }
4370
4371 static int
4372 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
4373                   unsigned scaler_user, int *scaler_id, unsigned int rotation,
4374                   int src_w, int src_h, int dst_w, int dst_h)
4375 {
4376         struct intel_crtc_scaler_state *scaler_state =
4377                 &crtc_state->scaler_state;
4378         struct intel_crtc *intel_crtc =
4379                 to_intel_crtc(crtc_state->base.crtc);
4380         int need_scaling;
4381
4382         need_scaling = intel_rotation_90_or_270(rotation) ?
4383                 (src_h != dst_w || src_w != dst_h):
4384                 (src_w != dst_w || src_h != dst_h);
4385
4386         /*
4387          * if plane is being disabled or scaler is no more required or force detach
4388          *  - free scaler binded to this plane/crtc
4389          *  - in order to do this, update crtc->scaler_usage
4390          *
4391          * Here scaler state in crtc_state is set free so that
4392          * scaler can be assigned to other user. Actual register
4393          * update to free the scaler is done in plane/panel-fit programming.
4394          * For this purpose crtc/plane_state->scaler_id isn't reset here.
4395          */
4396         if (force_detach || !need_scaling) {
4397                 if (*scaler_id >= 0) {
4398                         scaler_state->scaler_users &= ~(1 << scaler_user);
4399                         scaler_state->scalers[*scaler_id].in_use = 0;
4400
4401                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4402                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
4403                                 intel_crtc->pipe, scaler_user, *scaler_id,
4404                                 scaler_state->scaler_users);
4405                         *scaler_id = -1;
4406                 }
4407                 return 0;
4408         }
4409
4410         /* range checks */
4411         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
4412                 dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
4413
4414                 src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
4415                 dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H) {
4416                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
4417                         "size is out of scaler range\n",
4418                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
4419                 return -EINVAL;
4420         }
4421
4422         /* mark this plane as a scaler user in crtc_state */
4423         scaler_state->scaler_users |= (1 << scaler_user);
4424         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4425                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
4426                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
4427                 scaler_state->scaler_users);
4428
4429         return 0;
4430 }
4431
4432 /**
4433  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
4434  *
4435  * @state: crtc's scaler state
4436  *
4437  * Return
4438  *     0 - scaler_usage updated successfully
4439  *    error - requested scaling cannot be supported or other error condition
4440  */
4441 int skl_update_scaler_crtc(struct intel_crtc_state *state)
4442 {
4443         struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc);
4444         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
4445
4446         DRM_DEBUG_KMS("Updating scaler for [CRTC:%i] scaler_user index %u.%u\n",
4447                       intel_crtc->base.base.id, intel_crtc->pipe, SKL_CRTC_INDEX);
4448
4449         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
4450                 &state->scaler_state.scaler_id, BIT(DRM_ROTATE_0),
4451                 state->pipe_src_w, state->pipe_src_h,
4452                 adjusted_mode->crtc_hdisplay, adjusted_mode->crtc_vdisplay);
4453 }
4454
4455 /**
4456  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
4457  *
4458  * @state: crtc's scaler state
4459  * @plane_state: atomic plane state to update
4460  *
4461  * Return
4462  *     0 - scaler_usage updated successfully
4463  *    error - requested scaling cannot be supported or other error condition
4464  */
4465 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
4466                                    struct intel_plane_state *plane_state)
4467 {
4468
4469         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
4470         struct intel_plane *intel_plane =
4471                 to_intel_plane(plane_state->base.plane);
4472         struct drm_framebuffer *fb = plane_state->base.fb;
4473         int ret;
4474
4475         bool force_detach = !fb || !plane_state->visible;
4476
4477         DRM_DEBUG_KMS("Updating scaler for [PLANE:%d] scaler_user index %u.%u\n",
4478                       intel_plane->base.base.id, intel_crtc->pipe,
4479                       drm_plane_index(&intel_plane->base));
4480
4481         ret = skl_update_scaler(crtc_state, force_detach,
4482                                 drm_plane_index(&intel_plane->base),
4483                                 &plane_state->scaler_id,
4484                                 plane_state->base.rotation,
4485                                 drm_rect_width(&plane_state->src) >> 16,
4486                                 drm_rect_height(&plane_state->src) >> 16,
4487                                 drm_rect_width(&plane_state->dst),
4488                                 drm_rect_height(&plane_state->dst));
4489
4490         if (ret || plane_state->scaler_id < 0)
4491                 return ret;
4492
4493         /* check colorkey */
4494         if (plane_state->ckey.flags != I915_SET_COLORKEY_NONE) {
4495                 DRM_DEBUG_KMS("[PLANE:%d] scaling with color key not allowed",
4496                               intel_plane->base.base.id);
4497                 return -EINVAL;
4498         }
4499
4500         /* Check src format */
4501         switch (fb->pixel_format) {
4502         case DRM_FORMAT_RGB565:
4503         case DRM_FORMAT_XBGR8888:
4504         case DRM_FORMAT_XRGB8888:
4505         case DRM_FORMAT_ABGR8888:
4506         case DRM_FORMAT_ARGB8888:
4507         case DRM_FORMAT_XRGB2101010:
4508         case DRM_FORMAT_XBGR2101010:
4509         case DRM_FORMAT_YUYV:
4510         case DRM_FORMAT_YVYU:
4511         case DRM_FORMAT_UYVY:
4512         case DRM_FORMAT_VYUY:
4513                 break;
4514         default:
4515                 DRM_DEBUG_KMS("[PLANE:%d] FB:%d unsupported scaling format 0x%x\n",
4516                         intel_plane->base.base.id, fb->base.id, fb->pixel_format);
4517                 return -EINVAL;
4518         }
4519
4520         return 0;
4521 }
4522
4523 static void skylake_scaler_disable(struct intel_crtc *crtc)
4524 {
4525         int i;
4526
4527         for (i = 0; i < crtc->num_scalers; i++)
4528                 skl_detach_scaler(crtc, i);
4529 }
4530
4531 static void skylake_pfit_enable(struct intel_crtc *crtc)
4532 {
4533         struct drm_device *dev = crtc->base.dev;
4534         struct drm_i915_private *dev_priv = dev->dev_private;
4535         int pipe = crtc->pipe;
4536         struct intel_crtc_scaler_state *scaler_state =
4537                 &crtc->config->scaler_state;
4538
4539         DRM_DEBUG_KMS("for crtc_state = %p\n", crtc->config);
4540
4541         if (crtc->config->pch_pfit.enabled) {
4542                 int id;
4543
4544                 if (WARN_ON(crtc->config->scaler_state.scaler_id < 0)) {
4545                         DRM_ERROR("Requesting pfit without getting a scaler first\n");
4546                         return;
4547                 }
4548
4549                 id = scaler_state->scaler_id;
4550                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
4551                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
4552                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc->config->pch_pfit.pos);
4553                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc->config->pch_pfit.size);
4554
4555                 DRM_DEBUG_KMS("for crtc_state = %p scaler_id = %d\n", crtc->config, id);
4556         }
4557 }
4558
4559 static void ironlake_pfit_enable(struct intel_crtc *crtc)
4560 {
4561         struct drm_device *dev = crtc->base.dev;
4562         struct drm_i915_private *dev_priv = dev->dev_private;
4563         int pipe = crtc->pipe;
4564
4565         if (crtc->config->pch_pfit.enabled) {
4566                 /* Force use of hard-coded filter coefficients
4567                  * as some pre-programmed values are broken,
4568                  * e.g. x201.
4569                  */
4570                 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
4571                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
4572                                                  PF_PIPE_SEL_IVB(pipe));
4573                 else
4574                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
4575                 I915_WRITE(PF_WIN_POS(pipe), crtc->config->pch_pfit.pos);
4576                 I915_WRITE(PF_WIN_SZ(pipe), crtc->config->pch_pfit.size);
4577         }
4578 }
4579
4580 void hsw_enable_ips(struct intel_crtc *crtc)
4581 {
4582         struct drm_device *dev = crtc->base.dev;
4583         struct drm_i915_private *dev_priv = dev->dev_private;
4584
4585         if (!crtc->config->ips_enabled)
4586                 return;
4587
4588         /* We can only enable IPS after we enable a plane and wait for a vblank */
4589         intel_wait_for_vblank(dev, crtc->pipe);
4590
4591         assert_plane_enabled(dev_priv, crtc->plane);
4592         if (IS_BROADWELL(dev)) {
4593                 mutex_lock(&dev_priv->rps.hw_lock);
4594                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0xc0000000));
4595                 mutex_unlock(&dev_priv->rps.hw_lock);
4596                 /* Quoting Art Runyan: "its not safe to expect any particular
4597                  * value in IPS_CTL bit 31 after enabling IPS through the
4598                  * mailbox." Moreover, the mailbox may return a bogus state,
4599                  * so we need to just enable it and continue on.
4600                  */
4601         } else {
4602                 I915_WRITE(IPS_CTL, IPS_ENABLE);
4603                 /* The bit only becomes 1 in the next vblank, so this wait here
4604                  * is essentially intel_wait_for_vblank. If we don't have this
4605                  * and don't wait for vblanks until the end of crtc_enable, then
4606                  * the HW state readout code will complain that the expected
4607                  * IPS_CTL value is not the one we read. */
4608                 if (wait_for(I915_READ_NOTRACE(IPS_CTL) & IPS_ENABLE, 50))
4609                         DRM_ERROR("Timed out waiting for IPS enable\n");
4610         }
4611 }
4612
4613 void hsw_disable_ips(struct intel_crtc *crtc)
4614 {
4615         struct drm_device *dev = crtc->base.dev;
4616         struct drm_i915_private *dev_priv = dev->dev_private;
4617
4618         if (!crtc->config->ips_enabled)
4619                 return;
4620
4621         assert_plane_enabled(dev_priv, crtc->plane);
4622         if (IS_BROADWELL(dev)) {
4623                 mutex_lock(&dev_priv->rps.hw_lock);
4624                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
4625                 mutex_unlock(&dev_priv->rps.hw_lock);
4626                 /* wait for pcode to finish disabling IPS, which may take up to 42ms */
4627                 if (wait_for((I915_READ(IPS_CTL) & IPS_ENABLE) == 0, 42))
4628                         DRM_ERROR("Timed out waiting for IPS disable\n");
4629         } else {
4630                 I915_WRITE(IPS_CTL, 0);
4631                 POSTING_READ(IPS_CTL);
4632         }
4633
4634         /* We need to wait for a vblank before we can disable the plane. */
4635         intel_wait_for_vblank(dev, crtc->pipe);
4636 }
4637
4638 /** Loads the palette/gamma unit for the CRTC with the prepared values */
4639 static void intel_crtc_load_lut(struct drm_crtc *crtc)
4640 {
4641         struct drm_device *dev = crtc->dev;
4642         struct drm_i915_private *dev_priv = dev->dev_private;
4643         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4644         enum pipe pipe = intel_crtc->pipe;
4645         int i;
4646         bool reenable_ips = false;
4647
4648         /* The clocks have to be on to load the palette. */
4649         if (!crtc->state->active)
4650                 return;
4651
4652         if (HAS_GMCH_DISPLAY(dev_priv->dev)) {
4653                 if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI))
4654                         assert_dsi_pll_enabled(dev_priv);
4655                 else
4656                         assert_pll_enabled(dev_priv, pipe);
4657         }
4658
4659         /* Workaround : Do not read or write the pipe palette/gamma data while
4660          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
4661          */
4662         if (IS_HASWELL(dev) && intel_crtc->config->ips_enabled &&
4663             ((I915_READ(GAMMA_MODE(pipe)) & GAMMA_MODE_MODE_MASK) ==
4664              GAMMA_MODE_MODE_SPLIT)) {
4665                 hsw_disable_ips(intel_crtc);
4666                 reenable_ips = true;
4667         }
4668
4669         for (i = 0; i < 256; i++) {
4670                 u32 palreg;
4671
4672                 if (HAS_GMCH_DISPLAY(dev))
4673                         palreg = PALETTE(pipe, i);
4674                 else
4675                         palreg = LGC_PALETTE(pipe, i);
4676
4677                 I915_WRITE(palreg,
4678                            (intel_crtc->lut_r[i] << 16) |
4679                            (intel_crtc->lut_g[i] << 8) |
4680                            intel_crtc->lut_b[i]);
4681         }
4682
4683         if (reenable_ips)
4684                 hsw_enable_ips(intel_crtc);
4685 }
4686
4687 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
4688 {
4689         if (intel_crtc->overlay) {
4690                 struct drm_device *dev = intel_crtc->base.dev;
4691                 struct drm_i915_private *dev_priv = dev->dev_private;
4692
4693                 mutex_lock(&dev->struct_mutex);
4694                 dev_priv->mm.interruptible = false;
4695                 (void) intel_overlay_switch_off(intel_crtc->overlay);
4696                 dev_priv->mm.interruptible = true;
4697                 mutex_unlock(&dev->struct_mutex);
4698         }
4699
4700         /* Let userspace switch the overlay on again. In most cases userspace
4701          * has to recompute where to put it anyway.
4702          */
4703 }
4704
4705 /**
4706  * intel_post_enable_primary - Perform operations after enabling primary plane
4707  * @crtc: the CRTC whose primary plane was just enabled
4708  *
4709  * Performs potentially sleeping operations that must be done after the primary
4710  * plane is enabled, such as updating FBC and IPS.  Note that this may be
4711  * called due to an explicit primary plane update, or due to an implicit
4712  * re-enable that is caused when a sprite plane is updated to no longer
4713  * completely hide the primary plane.
4714  */
4715 static void
4716 intel_post_enable_primary(struct drm_crtc *crtc)
4717 {
4718         struct drm_device *dev = crtc->dev;
4719         struct drm_i915_private *dev_priv = dev->dev_private;
4720         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4721         int pipe = intel_crtc->pipe;
4722
4723         /*
4724          * BDW signals flip done immediately if the plane
4725          * is disabled, even if the plane enable is already
4726          * armed to occur at the next vblank :(
4727          */
4728         if (IS_BROADWELL(dev))
4729                 intel_wait_for_vblank(dev, pipe);
4730
4731         /*
4732          * FIXME IPS should be fine as long as one plane is
4733          * enabled, but in practice it seems to have problems
4734          * when going from primary only to sprite only and vice
4735          * versa.
4736          */
4737         hsw_enable_ips(intel_crtc);
4738
4739         /*
4740          * Gen2 reports pipe underruns whenever all planes are disabled.
4741          * So don't enable underrun reporting before at least some planes
4742          * are enabled.
4743          * FIXME: Need to fix the logic to work when we turn off all planes
4744          * but leave the pipe running.
4745          */
4746         if (IS_GEN2(dev))
4747                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
4748
4749         /* Underruns don't raise interrupts, so check manually. */
4750         if (HAS_GMCH_DISPLAY(dev))
4751                 i9xx_check_fifo_underruns(dev_priv);
4752 }
4753
4754 /**
4755  * intel_pre_disable_primary - Perform operations before disabling primary plane
4756  * @crtc: the CRTC whose primary plane is to be disabled
4757  *
4758  * Performs potentially sleeping operations that must be done before the
4759  * primary plane is disabled, such as updating FBC and IPS.  Note that this may
4760  * be called due to an explicit primary plane update, or due to an implicit
4761  * disable that is caused when a sprite plane completely hides the primary
4762  * plane.
4763  */
4764 static void
4765 intel_pre_disable_primary(struct drm_crtc *crtc)
4766 {
4767         struct drm_device *dev = crtc->dev;
4768         struct drm_i915_private *dev_priv = dev->dev_private;
4769         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4770         int pipe = intel_crtc->pipe;
4771
4772         /*
4773          * Gen2 reports pipe underruns whenever all planes are disabled.
4774          * So diasble underrun reporting before all the planes get disabled.
4775          * FIXME: Need to fix the logic to work when we turn off all planes
4776          * but leave the pipe running.
4777          */
4778         if (IS_GEN2(dev))
4779                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
4780
4781         /*
4782          * Vblank time updates from the shadow to live plane control register
4783          * are blocked if the memory self-refresh mode is active at that
4784          * moment. So to make sure the plane gets truly disabled, disable
4785          * first the self-refresh mode. The self-refresh enable bit in turn
4786          * will be checked/applied by the HW only at the next frame start
4787          * event which is after the vblank start event, so we need to have a
4788          * wait-for-vblank between disabling the plane and the pipe.
4789          */
4790         if (HAS_GMCH_DISPLAY(dev)) {
4791                 intel_set_memory_cxsr(dev_priv, false);
4792                 dev_priv->wm.vlv.cxsr = false;
4793                 intel_wait_for_vblank(dev, pipe);
4794         }
4795
4796         /*
4797          * FIXME IPS should be fine as long as one plane is
4798          * enabled, but in practice it seems to have problems
4799          * when going from primary only to sprite only and vice
4800          * versa.
4801          */
4802         hsw_disable_ips(intel_crtc);
4803 }
4804
4805 static void intel_post_plane_update(struct intel_crtc *crtc)
4806 {
4807         struct intel_crtc_atomic_commit *atomic = &crtc->atomic;
4808         struct drm_device *dev = crtc->base.dev;
4809         struct drm_i915_private *dev_priv = dev->dev_private;
4810         struct drm_plane *plane;
4811
4812         if (atomic->wait_vblank)
4813                 intel_wait_for_vblank(dev, crtc->pipe);
4814
4815         intel_frontbuffer_flip(dev, atomic->fb_bits);
4816
4817         if (atomic->disable_cxsr)
4818                 crtc->wm.cxsr_allowed = true;
4819
4820         if (crtc->atomic.update_wm_post)
4821                 intel_update_watermarks(&crtc->base);
4822
4823         if (atomic->update_fbc)
4824                 intel_fbc_update(dev_priv);
4825
4826         if (atomic->post_enable_primary)
4827                 intel_post_enable_primary(&crtc->base);
4828
4829         drm_for_each_plane_mask(plane, dev, atomic->update_sprite_watermarks)
4830                 intel_update_sprite_watermarks(plane, &crtc->base,
4831                                                0, 0, 0, false, false);
4832
4833         memset(atomic, 0, sizeof(*atomic));
4834 }
4835
4836 static void intel_pre_plane_update(struct intel_crtc *crtc)
4837 {
4838         struct drm_device *dev = crtc->base.dev;
4839         struct drm_i915_private *dev_priv = dev->dev_private;
4840         struct intel_crtc_atomic_commit *atomic = &crtc->atomic;
4841         struct drm_plane *p;
4842
4843         /* Track fb's for any planes being disabled */
4844         drm_for_each_plane_mask(p, dev, atomic->disabled_planes) {
4845                 struct intel_plane *plane = to_intel_plane(p);
4846
4847                 mutex_lock(&dev->struct_mutex);
4848                 i915_gem_track_fb(intel_fb_obj(plane->base.fb), NULL,
4849                                   plane->frontbuffer_bit);
4850                 mutex_unlock(&dev->struct_mutex);
4851         }
4852
4853         if (atomic->wait_for_flips)
4854                 intel_crtc_wait_for_pending_flips(&crtc->base);
4855
4856         if (atomic->disable_fbc)
4857                 intel_fbc_disable_crtc(crtc);
4858
4859         if (crtc->atomic.disable_ips)
4860                 hsw_disable_ips(crtc);
4861
4862         if (atomic->pre_disable_primary)
4863                 intel_pre_disable_primary(&crtc->base);
4864
4865         if (atomic->disable_cxsr) {
4866                 crtc->wm.cxsr_allowed = false;
4867                 intel_set_memory_cxsr(dev_priv, false);
4868         }
4869 }
4870
4871 static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask)
4872 {
4873         struct drm_device *dev = crtc->dev;
4874         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4875         struct drm_plane *p;
4876         int pipe = intel_crtc->pipe;
4877
4878         intel_crtc_dpms_overlay_disable(intel_crtc);
4879
4880         drm_for_each_plane_mask(p, dev, plane_mask)
4881                 to_intel_plane(p)->disable_plane(p, crtc);
4882
4883         /*
4884          * FIXME: Once we grow proper nuclear flip support out of this we need
4885          * to compute the mask of flip planes precisely. For the time being
4886          * consider this a flip to a NULL plane.
4887          */
4888         intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_ALL_MASK(pipe));
4889 }
4890
4891 static void ironlake_crtc_enable(struct drm_crtc *crtc)
4892 {
4893         struct drm_device *dev = crtc->dev;
4894         struct drm_i915_private *dev_priv = dev->dev_private;
4895         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4896         struct intel_encoder *encoder;
4897         int pipe = intel_crtc->pipe;
4898
4899         if (WARN_ON(intel_crtc->active))
4900                 return;
4901
4902         if (intel_crtc->config->has_pch_encoder)
4903                 intel_prepare_shared_dpll(intel_crtc);
4904
4905         if (intel_crtc->config->has_dp_encoder)
4906                 intel_dp_set_m_n(intel_crtc, M1_N1);
4907
4908         intel_set_pipe_timings(intel_crtc);
4909
4910         if (intel_crtc->config->has_pch_encoder) {
4911                 intel_cpu_transcoder_set_m_n(intel_crtc,
4912                                      &intel_crtc->config->fdi_m_n, NULL);
4913         }
4914
4915         ironlake_set_pipeconf(crtc);
4916
4917         intel_crtc->active = true;
4918
4919         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
4920         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
4921
4922         for_each_encoder_on_crtc(dev, crtc, encoder)
4923                 if (encoder->pre_enable)
4924                         encoder->pre_enable(encoder);
4925
4926         if (intel_crtc->config->has_pch_encoder) {
4927                 /* Note: FDI PLL enabling _must_ be done before we enable the
4928                  * cpu pipes, hence this is separate from all the other fdi/pch
4929                  * enabling. */
4930                 ironlake_fdi_pll_enable(intel_crtc);
4931         } else {
4932                 assert_fdi_tx_disabled(dev_priv, pipe);
4933                 assert_fdi_rx_disabled(dev_priv, pipe);
4934         }
4935
4936         ironlake_pfit_enable(intel_crtc);
4937
4938         /*
4939          * On ILK+ LUT must be loaded before the pipe is running but with
4940          * clocks enabled
4941          */
4942         intel_crtc_load_lut(crtc);
4943
4944         intel_update_watermarks(crtc);
4945         intel_enable_pipe(intel_crtc);
4946
4947         if (intel_crtc->config->has_pch_encoder)
4948                 ironlake_pch_enable(crtc);
4949
4950         assert_vblank_disabled(crtc);
4951         drm_crtc_vblank_on(crtc);
4952
4953         for_each_encoder_on_crtc(dev, crtc, encoder)
4954                 encoder->enable(encoder);
4955
4956         if (HAS_PCH_CPT(dev))
4957                 cpt_verify_modeset(dev, intel_crtc->pipe);
4958 }
4959
4960 /* IPS only exists on ULT machines and is tied to pipe A. */
4961 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
4962 {
4963         return HAS_IPS(crtc->base.dev) && crtc->pipe == PIPE_A;
4964 }
4965
4966 static void haswell_crtc_enable(struct drm_crtc *crtc)
4967 {
4968         struct drm_device *dev = crtc->dev;
4969         struct drm_i915_private *dev_priv = dev->dev_private;
4970         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4971         struct intel_encoder *encoder;
4972         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
4973         struct intel_crtc_state *pipe_config =
4974                 to_intel_crtc_state(crtc->state);
4975         bool is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
4976
4977         if (WARN_ON(intel_crtc->active))
4978                 return;
4979
4980         if (intel_crtc_to_shared_dpll(intel_crtc))
4981                 intel_enable_shared_dpll(intel_crtc);
4982
4983         if (intel_crtc->config->has_dp_encoder)
4984                 intel_dp_set_m_n(intel_crtc, M1_N1);
4985
4986         intel_set_pipe_timings(intel_crtc);
4987
4988         if (intel_crtc->config->cpu_transcoder != TRANSCODER_EDP) {
4989                 I915_WRITE(PIPE_MULT(intel_crtc->config->cpu_transcoder),
4990                            intel_crtc->config->pixel_multiplier - 1);
4991         }
4992
4993         if (intel_crtc->config->has_pch_encoder) {
4994                 intel_cpu_transcoder_set_m_n(intel_crtc,
4995                                      &intel_crtc->config->fdi_m_n, NULL);
4996         }
4997
4998         haswell_set_pipeconf(crtc);
4999
5000         intel_set_pipe_csc(crtc);
5001
5002         intel_crtc->active = true;
5003
5004         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5005         for_each_encoder_on_crtc(dev, crtc, encoder) {
5006                 if (encoder->pre_pll_enable)
5007                         encoder->pre_pll_enable(encoder);
5008                 if (encoder->pre_enable)
5009                         encoder->pre_enable(encoder);
5010         }
5011
5012         if (intel_crtc->config->has_pch_encoder) {
5013                 intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
5014                                                       true);
5015                 dev_priv->display.fdi_link_train(crtc);
5016         }
5017
5018         if (!is_dsi)
5019                 intel_ddi_enable_pipe_clock(intel_crtc);
5020
5021         if (INTEL_INFO(dev)->gen >= 9)
5022                 skylake_pfit_enable(intel_crtc);
5023         else
5024                 ironlake_pfit_enable(intel_crtc);
5025
5026         /*
5027          * On ILK+ LUT must be loaded before the pipe is running but with
5028          * clocks enabled
5029          */
5030         intel_crtc_load_lut(crtc);
5031
5032         intel_ddi_set_pipe_settings(crtc);
5033         if (!is_dsi)
5034                 intel_ddi_enable_transcoder_func(crtc);
5035
5036         intel_update_watermarks(crtc);
5037         intel_enable_pipe(intel_crtc);
5038
5039         if (intel_crtc->config->has_pch_encoder)
5040                 lpt_pch_enable(crtc);
5041
5042         if (intel_crtc->config->dp_encoder_is_mst && !is_dsi)
5043                 intel_ddi_set_vc_payload_alloc(crtc, true);
5044
5045         assert_vblank_disabled(crtc);
5046         drm_crtc_vblank_on(crtc);
5047
5048         for_each_encoder_on_crtc(dev, crtc, encoder) {
5049                 encoder->enable(encoder);
5050                 intel_opregion_notify_encoder(encoder, true);
5051         }
5052
5053         /* If we change the relative order between pipe/planes enabling, we need
5054          * to change the workaround. */
5055         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
5056         if (IS_HASWELL(dev) && hsw_workaround_pipe != INVALID_PIPE) {
5057                 intel_wait_for_vblank(dev, hsw_workaround_pipe);
5058                 intel_wait_for_vblank(dev, hsw_workaround_pipe);
5059         }
5060 }
5061
5062 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force)
5063 {
5064         struct drm_device *dev = crtc->base.dev;
5065         struct drm_i915_private *dev_priv = dev->dev_private;
5066         int pipe = crtc->pipe;
5067
5068         /* To avoid upsetting the power well on haswell only disable the pfit if
5069          * it's in use. The hw state code will make sure we get this right. */
5070         if (force || crtc->config->pch_pfit.enabled) {
5071                 I915_WRITE(PF_CTL(pipe), 0);
5072                 I915_WRITE(PF_WIN_POS(pipe), 0);
5073                 I915_WRITE(PF_WIN_SZ(pipe), 0);
5074         }
5075 }
5076
5077 static void ironlake_crtc_disable(struct drm_crtc *crtc)
5078 {
5079         struct drm_device *dev = crtc->dev;
5080         struct drm_i915_private *dev_priv = dev->dev_private;
5081         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5082         struct intel_encoder *encoder;
5083         int pipe = intel_crtc->pipe;
5084         u32 reg, temp;
5085
5086         for_each_encoder_on_crtc(dev, crtc, encoder)
5087                 encoder->disable(encoder);
5088
5089         drm_crtc_vblank_off(crtc);
5090         assert_vblank_disabled(crtc);
5091
5092         if (intel_crtc->config->has_pch_encoder)
5093                 intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5094
5095         intel_disable_pipe(intel_crtc);
5096
5097         ironlake_pfit_disable(intel_crtc, false);
5098
5099         if (intel_crtc->config->has_pch_encoder)
5100                 ironlake_fdi_disable(crtc);
5101
5102         for_each_encoder_on_crtc(dev, crtc, encoder)
5103                 if (encoder->post_disable)
5104                         encoder->post_disable(encoder);
5105
5106         if (intel_crtc->config->has_pch_encoder) {
5107                 ironlake_disable_pch_transcoder(dev_priv, pipe);
5108
5109                 if (HAS_PCH_CPT(dev)) {
5110                         /* disable TRANS_DP_CTL */
5111                         reg = TRANS_DP_CTL(pipe);
5112                         temp = I915_READ(reg);
5113                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
5114                                   TRANS_DP_PORT_SEL_MASK);
5115                         temp |= TRANS_DP_PORT_SEL_NONE;
5116                         I915_WRITE(reg, temp);
5117
5118                         /* disable DPLL_SEL */
5119                         temp = I915_READ(PCH_DPLL_SEL);
5120                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
5121                         I915_WRITE(PCH_DPLL_SEL, temp);
5122                 }
5123
5124                 ironlake_fdi_pll_disable(intel_crtc);
5125         }
5126 }
5127
5128 static void haswell_crtc_disable(struct drm_crtc *crtc)
5129 {
5130         struct drm_device *dev = crtc->dev;
5131         struct drm_i915_private *dev_priv = dev->dev_private;
5132         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5133         struct intel_encoder *encoder;
5134         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
5135         bool is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
5136
5137         for_each_encoder_on_crtc(dev, crtc, encoder) {
5138                 intel_opregion_notify_encoder(encoder, false);
5139                 encoder->disable(encoder);
5140         }
5141
5142         drm_crtc_vblank_off(crtc);
5143         assert_vblank_disabled(crtc);
5144
5145         if (intel_crtc->config->has_pch_encoder)
5146                 intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
5147                                                       false);
5148         intel_disable_pipe(intel_crtc);
5149
5150         if (intel_crtc->config->dp_encoder_is_mst)
5151                 intel_ddi_set_vc_payload_alloc(crtc, false);
5152
5153         if (!is_dsi)
5154                 intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
5155
5156         if (INTEL_INFO(dev)->gen >= 9)
5157                 skylake_scaler_disable(intel_crtc);
5158         else
5159                 ironlake_pfit_disable(intel_crtc, false);
5160
5161         if (!is_dsi)
5162                 intel_ddi_disable_pipe_clock(intel_crtc);
5163
5164         if (intel_crtc->config->has_pch_encoder) {
5165                 lpt_disable_pch_transcoder(dev_priv);
5166                 intel_ddi_fdi_disable(crtc);
5167         }
5168
5169         for_each_encoder_on_crtc(dev, crtc, encoder)
5170                 if (encoder->post_disable)
5171                         encoder->post_disable(encoder);
5172 }
5173
5174 static void i9xx_pfit_enable(struct intel_crtc *crtc)
5175 {
5176         struct drm_device *dev = crtc->base.dev;
5177         struct drm_i915_private *dev_priv = dev->dev_private;
5178         struct intel_crtc_state *pipe_config = crtc->config;
5179
5180         if (!pipe_config->gmch_pfit.control)
5181                 return;
5182
5183         /*
5184          * The panel fitter should only be adjusted whilst the pipe is disabled,
5185          * according to register description and PRM.
5186          */
5187         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
5188         assert_pipe_disabled(dev_priv, crtc->pipe);
5189
5190         I915_WRITE(PFIT_PGM_RATIOS, pipe_config->gmch_pfit.pgm_ratios);
5191         I915_WRITE(PFIT_CONTROL, pipe_config->gmch_pfit.control);
5192
5193         /* Border color in case we don't scale up to the full screen. Black by
5194          * default, change to something else for debugging. */
5195         I915_WRITE(BCLRPAT(crtc->pipe), 0);
5196 }
5197
5198 static enum intel_display_power_domain port_to_power_domain(enum port port)
5199 {
5200         switch (port) {
5201         case PORT_A:
5202                 return POWER_DOMAIN_PORT_DDI_A_4_LANES;
5203         case PORT_B:
5204                 return POWER_DOMAIN_PORT_DDI_B_4_LANES;
5205         case PORT_C:
5206                 return POWER_DOMAIN_PORT_DDI_C_4_LANES;
5207         case PORT_D:
5208                 return POWER_DOMAIN_PORT_DDI_D_4_LANES;
5209         case PORT_E:
5210                 return POWER_DOMAIN_PORT_DDI_E_2_LANES;
5211         default:
5212                 MISSING_CASE(port);
5213                 return POWER_DOMAIN_PORT_OTHER;
5214         }
5215 }
5216
5217 static enum intel_display_power_domain port_to_aux_power_domain(enum port port)
5218 {
5219         switch (port) {
5220         case PORT_A:
5221                 return POWER_DOMAIN_AUX_A;
5222         case PORT_B:
5223                 return POWER_DOMAIN_AUX_B;
5224         case PORT_C:
5225                 return POWER_DOMAIN_AUX_C;
5226         case PORT_D:
5227                 return POWER_DOMAIN_AUX_D;
5228         case PORT_E:
5229                 /* FIXME: Check VBT for actual wiring of PORT E */
5230                 return POWER_DOMAIN_AUX_D;
5231         default:
5232                 MISSING_CASE(port);
5233                 return POWER_DOMAIN_AUX_A;
5234         }
5235 }
5236
5237 #define for_each_power_domain(domain, mask)                             \
5238         for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++)     \
5239                 if ((1 << (domain)) & (mask))
5240
5241 enum intel_display_power_domain
5242 intel_display_port_power_domain(struct intel_encoder *intel_encoder)
5243 {
5244         struct drm_device *dev = intel_encoder->base.dev;
5245         struct intel_digital_port *intel_dig_port;
5246
5247         switch (intel_encoder->type) {
5248         case INTEL_OUTPUT_UNKNOWN:
5249                 /* Only DDI platforms should ever use this output type */
5250                 WARN_ON_ONCE(!HAS_DDI(dev));
5251         case INTEL_OUTPUT_DISPLAYPORT:
5252         case INTEL_OUTPUT_HDMI:
5253         case INTEL_OUTPUT_EDP:
5254                 intel_dig_port = enc_to_dig_port(&intel_encoder->base);
5255                 return port_to_power_domain(intel_dig_port->port);
5256         case INTEL_OUTPUT_DP_MST:
5257                 intel_dig_port = enc_to_mst(&intel_encoder->base)->primary;
5258                 return port_to_power_domain(intel_dig_port->port);
5259         case INTEL_OUTPUT_ANALOG:
5260                 return POWER_DOMAIN_PORT_CRT;
5261         case INTEL_OUTPUT_DSI:
5262                 return POWER_DOMAIN_PORT_DSI;
5263         default:
5264                 return POWER_DOMAIN_PORT_OTHER;
5265         }
5266 }
5267
5268 enum intel_display_power_domain
5269 intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder)
5270 {
5271         struct drm_device *dev = intel_encoder->base.dev;
5272         struct intel_digital_port *intel_dig_port;
5273
5274         switch (intel_encoder->type) {
5275         case INTEL_OUTPUT_UNKNOWN:
5276         case INTEL_OUTPUT_HDMI:
5277                 /*
5278                  * Only DDI platforms should ever use these output types.
5279                  * We can get here after the HDMI detect code has already set
5280                  * the type of the shared encoder. Since we can't be sure
5281                  * what's the status of the given connectors, play safe and
5282                  * run the DP detection too.
5283                  */
5284                 WARN_ON_ONCE(!HAS_DDI(dev));
5285         case INTEL_OUTPUT_DISPLAYPORT:
5286         case INTEL_OUTPUT_EDP:
5287                 intel_dig_port = enc_to_dig_port(&intel_encoder->base);
5288                 return port_to_aux_power_domain(intel_dig_port->port);
5289         case INTEL_OUTPUT_DP_MST:
5290                 intel_dig_port = enc_to_mst(&intel_encoder->base)->primary;
5291                 return port_to_aux_power_domain(intel_dig_port->port);
5292         default:
5293                 MISSING_CASE(intel_encoder->type);
5294                 return POWER_DOMAIN_AUX_A;
5295         }
5296 }
5297
5298 static unsigned long get_crtc_power_domains(struct drm_crtc *crtc)
5299 {
5300         struct drm_device *dev = crtc->dev;
5301         struct intel_encoder *intel_encoder;
5302         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5303         enum pipe pipe = intel_crtc->pipe;
5304         unsigned long mask;
5305         enum transcoder transcoder;
5306
5307         if (!crtc->state->active)
5308                 return 0;
5309
5310         transcoder = intel_pipe_to_cpu_transcoder(dev->dev_private, pipe);
5311
5312         mask = BIT(POWER_DOMAIN_PIPE(pipe));
5313         mask |= BIT(POWER_DOMAIN_TRANSCODER(transcoder));
5314         if (intel_crtc->config->pch_pfit.enabled ||
5315             intel_crtc->config->pch_pfit.force_thru)
5316                 mask |= BIT(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
5317
5318         for_each_encoder_on_crtc(dev, crtc, intel_encoder)
5319                 mask |= BIT(intel_display_port_power_domain(intel_encoder));
5320
5321         return mask;
5322 }
5323
5324 static unsigned long modeset_get_crtc_power_domains(struct drm_crtc *crtc)
5325 {
5326         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
5327         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5328         enum intel_display_power_domain domain;
5329         unsigned long domains, new_domains, old_domains;
5330
5331         old_domains = intel_crtc->enabled_power_domains;
5332         intel_crtc->enabled_power_domains = new_domains = get_crtc_power_domains(crtc);
5333
5334         domains = new_domains & ~old_domains;
5335
5336         for_each_power_domain(domain, domains)
5337                 intel_display_power_get(dev_priv, domain);
5338
5339         return old_domains & ~new_domains;
5340 }
5341
5342 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
5343                                       unsigned long domains)
5344 {
5345         enum intel_display_power_domain domain;
5346
5347         for_each_power_domain(domain, domains)
5348                 intel_display_power_put(dev_priv, domain);
5349 }
5350
5351 static void modeset_update_crtc_power_domains(struct drm_atomic_state *state)
5352 {
5353         struct drm_device *dev = state->dev;
5354         struct drm_i915_private *dev_priv = dev->dev_private;
5355         unsigned long put_domains[I915_MAX_PIPES] = {};
5356         struct drm_crtc_state *crtc_state;
5357         struct drm_crtc *crtc;
5358         int i;
5359
5360         for_each_crtc_in_state(state, crtc, crtc_state, i) {
5361                 if (needs_modeset(crtc->state))
5362                         put_domains[to_intel_crtc(crtc)->pipe] =
5363                                 modeset_get_crtc_power_domains(crtc);
5364         }
5365
5366         if (dev_priv->display.modeset_commit_cdclk) {
5367                 unsigned int cdclk = to_intel_atomic_state(state)->cdclk;
5368
5369                 if (cdclk != dev_priv->cdclk_freq &&
5370                     !WARN_ON(!state->allow_modeset))
5371                         dev_priv->display.modeset_commit_cdclk(state);
5372         }
5373
5374         for (i = 0; i < I915_MAX_PIPES; i++)
5375                 if (put_domains[i])
5376                         modeset_put_power_domains(dev_priv, put_domains[i]);
5377 }
5378
5379 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
5380 {
5381         int max_cdclk_freq = dev_priv->max_cdclk_freq;
5382
5383         if (INTEL_INFO(dev_priv)->gen >= 9 ||
5384             IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5385                 return max_cdclk_freq;
5386         else if (IS_CHERRYVIEW(dev_priv))
5387                 return max_cdclk_freq*95/100;
5388         else if (INTEL_INFO(dev_priv)->gen < 4)
5389                 return 2*max_cdclk_freq*90/100;
5390         else
5391                 return max_cdclk_freq*90/100;
5392 }
5393
5394 static void intel_update_max_cdclk(struct drm_device *dev)
5395 {
5396         struct drm_i915_private *dev_priv = dev->dev_private;
5397
5398         if (IS_SKYLAKE(dev)) {
5399                 u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
5400
5401                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
5402                         dev_priv->max_cdclk_freq = 675000;
5403                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
5404                         dev_priv->max_cdclk_freq = 540000;
5405                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
5406                         dev_priv->max_cdclk_freq = 450000;
5407                 else
5408                         dev_priv->max_cdclk_freq = 337500;
5409         } else if (IS_BROADWELL(dev))  {
5410                 /*
5411                  * FIXME with extra cooling we can allow
5412                  * 540 MHz for ULX and 675 Mhz for ULT.
5413                  * How can we know if extra cooling is
5414                  * available? PCI ID, VTB, something else?
5415                  */
5416                 if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
5417                         dev_priv->max_cdclk_freq = 450000;
5418                 else if (IS_BDW_ULX(dev))
5419                         dev_priv->max_cdclk_freq = 450000;
5420                 else if (IS_BDW_ULT(dev))
5421                         dev_priv->max_cdclk_freq = 540000;
5422                 else
5423                         dev_priv->max_cdclk_freq = 675000;
5424         } else if (IS_CHERRYVIEW(dev)) {
5425                 dev_priv->max_cdclk_freq = 320000;
5426         } else if (IS_VALLEYVIEW(dev)) {
5427                 dev_priv->max_cdclk_freq = 400000;
5428         } else {
5429                 /* otherwise assume cdclk is fixed */
5430                 dev_priv->max_cdclk_freq = dev_priv->cdclk_freq;
5431         }
5432
5433         dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
5434
5435         DRM_DEBUG_DRIVER("Max CD clock rate: %d kHz\n",
5436                          dev_priv->max_cdclk_freq);
5437
5438         DRM_DEBUG_DRIVER("Max dotclock rate: %d kHz\n",
5439                          dev_priv->max_dotclk_freq);
5440 }
5441
5442 static void intel_update_cdclk(struct drm_device *dev)
5443 {
5444         struct drm_i915_private *dev_priv = dev->dev_private;
5445
5446         dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev);
5447         DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz\n",
5448                          dev_priv->cdclk_freq);
5449
5450         /*
5451          * Program the gmbus_freq based on the cdclk frequency.
5452          * BSpec erroneously claims we should aim for 4MHz, but
5453          * in fact 1MHz is the correct frequency.
5454          */
5455         if (IS_VALLEYVIEW(dev)) {
5456                 /*
5457                  * Program the gmbus_freq based on the cdclk frequency.
5458                  * BSpec erroneously claims we should aim for 4MHz, but
5459                  * in fact 1MHz is the correct frequency.
5460                  */
5461                 I915_WRITE(GMBUSFREQ_VLV, DIV_ROUND_UP(dev_priv->cdclk_freq, 1000));
5462         }
5463
5464         if (dev_priv->max_cdclk_freq == 0)
5465                 intel_update_max_cdclk(dev);
5466 }
5467
5468 static void broxton_set_cdclk(struct drm_device *dev, int frequency)
5469 {
5470         struct drm_i915_private *dev_priv = dev->dev_private;
5471         uint32_t divider;
5472         uint32_t ratio;
5473         uint32_t current_freq;
5474         int ret;
5475
5476         /* frequency = 19.2MHz * ratio / 2 / div{1,1.5,2,4} */
5477         switch (frequency) {
5478         case 144000:
5479                 divider = BXT_CDCLK_CD2X_DIV_SEL_4;
5480                 ratio = BXT_DE_PLL_RATIO(60);
5481                 break;
5482         case 288000:
5483                 divider = BXT_CDCLK_CD2X_DIV_SEL_2;
5484                 ratio = BXT_DE_PLL_RATIO(60);
5485                 break;
5486         case 384000:
5487                 divider = BXT_CDCLK_CD2X_DIV_SEL_1_5;
5488                 ratio = BXT_DE_PLL_RATIO(60);
5489                 break;
5490         case 576000:
5491                 divider = BXT_CDCLK_CD2X_DIV_SEL_1;
5492                 ratio = BXT_DE_PLL_RATIO(60);
5493                 break;
5494         case 624000:
5495                 divider = BXT_CDCLK_CD2X_DIV_SEL_1;
5496                 ratio = BXT_DE_PLL_RATIO(65);
5497                 break;
5498         case 19200:
5499                 /*
5500                  * Bypass frequency with DE PLL disabled. Init ratio, divider
5501                  * to suppress GCC warning.
5502                  */
5503                 ratio = 0;
5504                 divider = 0;
5505                 break;
5506         default:
5507                 DRM_ERROR("unsupported CDCLK freq %d", frequency);
5508
5509                 return;
5510         }
5511
5512         mutex_lock(&dev_priv->rps.hw_lock);
5513         /* Inform power controller of upcoming frequency change */
5514         ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
5515                                       0x80000000);
5516         mutex_unlock(&dev_priv->rps.hw_lock);
5517
5518         if (ret) {
5519                 DRM_ERROR("PCode CDCLK freq change notify failed (err %d, freq %d)\n",
5520                           ret, frequency);
5521                 return;
5522         }
5523
5524         current_freq = I915_READ(CDCLK_CTL) & CDCLK_FREQ_DECIMAL_MASK;
5525         /* convert from .1 fixpoint MHz with -1MHz offset to kHz */
5526         current_freq = current_freq * 500 + 1000;
5527
5528         /*
5529          * DE PLL has to be disabled when
5530          * - setting to 19.2MHz (bypass, PLL isn't used)
5531          * - before setting to 624MHz (PLL needs toggling)
5532          * - before setting to any frequency from 624MHz (PLL needs toggling)
5533          */
5534         if (frequency == 19200 || frequency == 624000 ||
5535             current_freq == 624000) {
5536                 I915_WRITE(BXT_DE_PLL_ENABLE, ~BXT_DE_PLL_PLL_ENABLE);
5537                 /* Timeout 200us */
5538                 if (wait_for(!(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK),
5539                              1))
5540                         DRM_ERROR("timout waiting for DE PLL unlock\n");
5541         }
5542
5543         if (frequency != 19200) {
5544                 uint32_t val;
5545
5546                 val = I915_READ(BXT_DE_PLL_CTL);
5547                 val &= ~BXT_DE_PLL_RATIO_MASK;
5548                 val |= ratio;
5549                 I915_WRITE(BXT_DE_PLL_CTL, val);
5550
5551                 I915_WRITE(BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
5552                 /* Timeout 200us */
5553                 if (wait_for(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK, 1))
5554                         DRM_ERROR("timeout waiting for DE PLL lock\n");
5555
5556                 val = I915_READ(CDCLK_CTL);
5557                 val &= ~BXT_CDCLK_CD2X_DIV_SEL_MASK;
5558                 val |= divider;
5559                 /*
5560                  * Disable SSA Precharge when CD clock frequency < 500 MHz,
5561                  * enable otherwise.
5562                  */
5563                 val &= ~BXT_CDCLK_SSA_PRECHARGE_ENABLE;
5564                 if (frequency >= 500000)
5565                         val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
5566
5567                 val &= ~CDCLK_FREQ_DECIMAL_MASK;
5568                 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
5569                 val |= (frequency - 1000) / 500;
5570                 I915_WRITE(CDCLK_CTL, val);
5571         }
5572
5573         mutex_lock(&dev_priv->rps.hw_lock);
5574         ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
5575                                       DIV_ROUND_UP(frequency, 25000));
5576         mutex_unlock(&dev_priv->rps.hw_lock);
5577
5578         if (ret) {
5579                 DRM_ERROR("PCode CDCLK freq set failed, (err %d, freq %d)\n",
5580                           ret, frequency);
5581                 return;
5582         }
5583
5584         intel_update_cdclk(dev);
5585 }
5586
5587 void broxton_init_cdclk(struct drm_device *dev)
5588 {
5589         struct drm_i915_private *dev_priv = dev->dev_private;
5590         uint32_t val;
5591
5592         /*
5593          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
5594          * or else the reset will hang because there is no PCH to respond.
5595          * Move the handshake programming to initialization sequence.
5596          * Previously was left up to BIOS.
5597          */
5598         val = I915_READ(HSW_NDE_RSTWRN_OPT);
5599         val &= ~RESET_PCH_HANDSHAKE_ENABLE;
5600         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
5601
5602         /* Enable PG1 for cdclk */
5603         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
5604
5605         /* check if cd clock is enabled */
5606         if (I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_PLL_ENABLE) {
5607                 DRM_DEBUG_KMS("Display already initialized\n");
5608                 return;
5609         }
5610
5611         /*
5612          * FIXME:
5613          * - The initial CDCLK needs to be read from VBT.
5614          *   Need to make this change after VBT has changes for BXT.
5615          * - check if setting the max (or any) cdclk freq is really necessary
5616          *   here, it belongs to modeset time
5617          */
5618         broxton_set_cdclk(dev, 624000);
5619
5620         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
5621         POSTING_READ(DBUF_CTL);
5622
5623         udelay(10);
5624
5625         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
5626                 DRM_ERROR("DBuf power enable timeout!\n");
5627 }
5628
5629 void broxton_uninit_cdclk(struct drm_device *dev)
5630 {
5631         struct drm_i915_private *dev_priv = dev->dev_private;
5632
5633         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
5634         POSTING_READ(DBUF_CTL);
5635
5636         udelay(10);
5637
5638         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
5639                 DRM_ERROR("DBuf power disable timeout!\n");
5640
5641         /* Set minimum (bypass) frequency, in effect turning off the DE PLL */
5642         broxton_set_cdclk(dev, 19200);
5643
5644         intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
5645 }
5646
5647 static const struct skl_cdclk_entry {
5648         unsigned int freq;
5649         unsigned int vco;
5650 } skl_cdclk_frequencies[] = {
5651         { .freq = 308570, .vco = 8640 },
5652         { .freq = 337500, .vco = 8100 },
5653         { .freq = 432000, .vco = 8640 },
5654         { .freq = 450000, .vco = 8100 },
5655         { .freq = 540000, .vco = 8100 },
5656         { .freq = 617140, .vco = 8640 },
5657         { .freq = 675000, .vco = 8100 },
5658 };
5659
5660 static unsigned int skl_cdclk_decimal(unsigned int freq)
5661 {
5662         return (freq - 1000) / 500;
5663 }
5664
5665 static unsigned int skl_cdclk_get_vco(unsigned int freq)
5666 {
5667         unsigned int i;
5668
5669         for (i = 0; i < ARRAY_SIZE(skl_cdclk_frequencies); i++) {
5670                 const struct skl_cdclk_entry *e = &skl_cdclk_frequencies[i];
5671
5672                 if (e->freq == freq)
5673                         return e->vco;
5674         }
5675
5676         return 8100;
5677 }
5678
5679 static void
5680 skl_dpll0_enable(struct drm_i915_private *dev_priv, unsigned int required_vco)
5681 {
5682         unsigned int min_freq;
5683         u32 val;
5684
5685         /* select the minimum CDCLK before enabling DPLL 0 */
5686         val = I915_READ(CDCLK_CTL);
5687         val &= ~CDCLK_FREQ_SEL_MASK | ~CDCLK_FREQ_DECIMAL_MASK;
5688         val |= CDCLK_FREQ_337_308;
5689
5690         if (required_vco == 8640)
5691                 min_freq = 308570;
5692         else
5693                 min_freq = 337500;
5694
5695         val = CDCLK_FREQ_337_308 | skl_cdclk_decimal(min_freq);
5696
5697         I915_WRITE(CDCLK_CTL, val);
5698         POSTING_READ(CDCLK_CTL);
5699
5700         /*
5701          * We always enable DPLL0 with the lowest link rate possible, but still
5702          * taking into account the VCO required to operate the eDP panel at the
5703          * desired frequency. The usual DP link rates operate with a VCO of
5704          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
5705          * The modeset code is responsible for the selection of the exact link
5706          * rate later on, with the constraint of choosing a frequency that
5707          * works with required_vco.
5708          */
5709         val = I915_READ(DPLL_CTRL1);
5710
5711         val &= ~(DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | DPLL_CTRL1_SSC(SKL_DPLL0) |
5712                  DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
5713         val |= DPLL_CTRL1_OVERRIDE(SKL_DPLL0);
5714         if (required_vco == 8640)
5715                 val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080,
5716                                             SKL_DPLL0);
5717         else
5718                 val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810,
5719                                             SKL_DPLL0);
5720
5721         I915_WRITE(DPLL_CTRL1, val);
5722         POSTING_READ(DPLL_CTRL1);
5723
5724         I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) | LCPLL_PLL_ENABLE);
5725
5726         if (wait_for(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK, 5))
5727                 DRM_ERROR("DPLL0 not locked\n");
5728 }
5729
5730 static bool skl_cdclk_pcu_ready(struct drm_i915_private *dev_priv)
5731 {
5732         int ret;
5733         u32 val;
5734
5735         /* inform PCU we want to change CDCLK */
5736         val = SKL_CDCLK_PREPARE_FOR_CHANGE;
5737         mutex_lock(&dev_priv->rps.hw_lock);
5738         ret = sandybridge_pcode_read(dev_priv, SKL_PCODE_CDCLK_CONTROL, &val);
5739         mutex_unlock(&dev_priv->rps.hw_lock);
5740
5741         return ret == 0 && (val & SKL_CDCLK_READY_FOR_CHANGE);
5742 }
5743
5744 static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv)
5745 {
5746         unsigned int i;
5747
5748         for (i = 0; i < 15; i++) {
5749                 if (skl_cdclk_pcu_ready(dev_priv))
5750                         return true;
5751                 udelay(10);
5752         }
5753
5754         return false;
5755 }
5756
5757 static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq)
5758 {
5759         struct drm_device *dev = dev_priv->dev;
5760         u32 freq_select, pcu_ack;
5761
5762         DRM_DEBUG_DRIVER("Changing CDCLK to %dKHz\n", freq);
5763
5764         if (!skl_cdclk_wait_for_pcu_ready(dev_priv)) {
5765                 DRM_ERROR("failed to inform PCU about cdclk change\n");
5766                 return;
5767         }
5768
5769         /* set CDCLK_CTL */
5770         switch(freq) {
5771         case 450000:
5772         case 432000:
5773                 freq_select = CDCLK_FREQ_450_432;
5774                 pcu_ack = 1;
5775                 break;
5776         case 540000:
5777                 freq_select = CDCLK_FREQ_540;
5778                 pcu_ack = 2;
5779                 break;
5780         case 308570:
5781         case 337500:
5782         default:
5783                 freq_select = CDCLK_FREQ_337_308;
5784                 pcu_ack = 0;
5785                 break;
5786         case 617140:
5787         case 675000:
5788                 freq_select = CDCLK_FREQ_675_617;
5789                 pcu_ack = 3;
5790                 break;
5791         }
5792
5793         I915_WRITE(CDCLK_CTL, freq_select | skl_cdclk_decimal(freq));
5794         POSTING_READ(CDCLK_CTL);
5795
5796         /* inform PCU of the change */
5797         mutex_lock(&dev_priv->rps.hw_lock);
5798         sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack);
5799         mutex_unlock(&dev_priv->rps.hw_lock);
5800
5801         intel_update_cdclk(dev);
5802 }
5803
5804 void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
5805 {
5806         /* disable DBUF power */
5807         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
5808         POSTING_READ(DBUF_CTL);
5809
5810         udelay(10);
5811
5812         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
5813                 DRM_ERROR("DBuf power disable timeout\n");
5814
5815         /*
5816          * DMC assumes ownership of LCPLL and will get confused if we touch it.
5817          */
5818         if (dev_priv->csr.dmc_payload) {
5819                 /* disable DPLL0 */
5820                 I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) &
5821                                         ~LCPLL_PLL_ENABLE);
5822                 if (wait_for(!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK), 1))
5823                         DRM_ERROR("Couldn't disable DPLL0\n");
5824         }
5825
5826         intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
5827 }
5828
5829 void skl_init_cdclk(struct drm_i915_private *dev_priv)
5830 {
5831         u32 val;
5832         unsigned int required_vco;
5833
5834         /* enable PCH reset handshake */
5835         val = I915_READ(HSW_NDE_RSTWRN_OPT);
5836         I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
5837
5838         /* enable PG1 and Misc I/O */
5839         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
5840
5841         /* DPLL0 not enabled (happens on early BIOS versions) */
5842         if (!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE)) {
5843                 /* enable DPLL0 */
5844                 required_vco = skl_cdclk_get_vco(dev_priv->skl_boot_cdclk);
5845                 skl_dpll0_enable(dev_priv, required_vco);
5846         }
5847
5848         /* set CDCLK to the frequency the BIOS chose */
5849         skl_set_cdclk(dev_priv, dev_priv->skl_boot_cdclk);
5850
5851         /* enable DBUF power */
5852         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
5853         POSTING_READ(DBUF_CTL);
5854
5855         udelay(10);
5856
5857         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
5858                 DRM_ERROR("DBuf power enable timeout\n");
5859 }
5860
5861 /* Adjust CDclk dividers to allow high res or save power if possible */
5862 static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
5863 {
5864         struct drm_i915_private *dev_priv = dev->dev_private;
5865         u32 val, cmd;
5866
5867         WARN_ON(dev_priv->display.get_display_clock_speed(dev)
5868                                         != dev_priv->cdclk_freq);
5869
5870         if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
5871                 cmd = 2;
5872         else if (cdclk == 266667)
5873                 cmd = 1;
5874         else
5875                 cmd = 0;
5876
5877         mutex_lock(&dev_priv->rps.hw_lock);
5878         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5879         val &= ~DSPFREQGUAR_MASK;
5880         val |= (cmd << DSPFREQGUAR_SHIFT);
5881         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
5882         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
5883                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
5884                      50)) {
5885                 DRM_ERROR("timed out waiting for CDclk change\n");
5886         }
5887         mutex_unlock(&dev_priv->rps.hw_lock);
5888
5889         mutex_lock(&dev_priv->sb_lock);
5890
5891         if (cdclk == 400000) {
5892                 u32 divider;
5893
5894                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
5895
5896                 /* adjust cdclk divider */
5897                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
5898                 val &= ~CCK_FREQUENCY_VALUES;
5899                 val |= divider;
5900                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
5901
5902                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
5903                               CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
5904                              50))
5905                         DRM_ERROR("timed out waiting for CDclk change\n");
5906         }
5907
5908         /* adjust self-refresh exit latency value */
5909         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
5910         val &= ~0x7f;
5911
5912         /*
5913          * For high bandwidth configs, we set a higher latency in the bunit
5914          * so that the core display fetch happens in time to avoid underruns.
5915          */
5916         if (cdclk == 400000)
5917                 val |= 4500 / 250; /* 4.5 usec */
5918         else
5919                 val |= 3000 / 250; /* 3.0 usec */
5920         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
5921
5922         mutex_unlock(&dev_priv->sb_lock);
5923
5924         intel_update_cdclk(dev);
5925 }
5926
5927 static void cherryview_set_cdclk(struct drm_device *dev, int cdclk)
5928 {
5929         struct drm_i915_private *dev_priv = dev->dev_private;
5930         u32 val, cmd;
5931
5932         WARN_ON(dev_priv->display.get_display_clock_speed(dev)
5933                                                 != dev_priv->cdclk_freq);
5934
5935         switch (cdclk) {
5936         case 333333:
5937         case 320000:
5938         case 266667:
5939         case 200000:
5940                 break;
5941         default:
5942                 MISSING_CASE(cdclk);
5943                 return;
5944         }
5945
5946         /*
5947          * Specs are full of misinformation, but testing on actual
5948          * hardware has shown that we just need to write the desired
5949          * CCK divider into the Punit register.
5950          */
5951         cmd = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
5952
5953         mutex_lock(&dev_priv->rps.hw_lock);
5954         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5955         val &= ~DSPFREQGUAR_MASK_CHV;
5956         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
5957         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
5958         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
5959                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
5960                      50)) {
5961                 DRM_ERROR("timed out waiting for CDclk change\n");
5962         }
5963         mutex_unlock(&dev_priv->rps.hw_lock);
5964
5965         intel_update_cdclk(dev);
5966 }
5967
5968 static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv,
5969                                  int max_pixclk)
5970 {
5971         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ? 333333 : 320000;
5972         int limit = IS_CHERRYVIEW(dev_priv) ? 95 : 90;
5973
5974         /*
5975          * Really only a few cases to deal with, as only 4 CDclks are supported:
5976          *   200MHz
5977          *   267MHz
5978          *   320/333MHz (depends on HPLL freq)
5979          *   400MHz (VLV only)
5980          * So we check to see whether we're above 90% (VLV) or 95% (CHV)
5981          * of the lower bin and adjust if needed.
5982          *
5983          * We seem to get an unstable or solid color picture at 200MHz.
5984          * Not sure what's wrong. For now use 200MHz only when all pipes
5985          * are off.
5986          */
5987         if (!IS_CHERRYVIEW(dev_priv) &&
5988             max_pixclk > freq_320*limit/100)
5989                 return 400000;
5990         else if (max_pixclk > 266667*limit/100)
5991                 return freq_320;
5992         else if (max_pixclk > 0)
5993                 return 266667;
5994         else
5995                 return 200000;
5996 }
5997
5998 static int broxton_calc_cdclk(struct drm_i915_private *dev_priv,
5999                               int max_pixclk)
6000 {
6001         /*
6002          * FIXME:
6003          * - remove the guardband, it's not needed on BXT
6004          * - set 19.2MHz bypass frequency if there are no active pipes
6005          */
6006         if (max_pixclk > 576000*9/10)
6007                 return 624000;
6008         else if (max_pixclk > 384000*9/10)
6009                 return 576000;
6010         else if (max_pixclk > 288000*9/10)
6011                 return 384000;
6012         else if (max_pixclk > 144000*9/10)
6013                 return 288000;
6014         else
6015                 return 144000;
6016 }
6017
6018 /* Compute the max pixel clock for new configuration. Uses atomic state if
6019  * that's non-NULL, look at current state otherwise. */
6020 static int intel_mode_max_pixclk(struct drm_device *dev,
6021                                  struct drm_atomic_state *state)
6022 {
6023         struct intel_crtc *intel_crtc;
6024         struct intel_crtc_state *crtc_state;
6025         int max_pixclk = 0;
6026
6027         for_each_intel_crtc(dev, intel_crtc) {
6028                 crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
6029                 if (IS_ERR(crtc_state))
6030                         return PTR_ERR(crtc_state);
6031
6032                 if (!crtc_state->base.enable)
6033                         continue;
6034
6035                 max_pixclk = max(max_pixclk,
6036                                  crtc_state->base.adjusted_mode.crtc_clock);
6037         }
6038
6039         return max_pixclk;
6040 }
6041
6042 static int valleyview_modeset_calc_cdclk(struct drm_atomic_state *state)
6043 {
6044         struct drm_device *dev = state->dev;
6045         struct drm_i915_private *dev_priv = dev->dev_private;
6046         int max_pixclk = intel_mode_max_pixclk(dev, state);
6047
6048         if (max_pixclk < 0)
6049                 return max_pixclk;
6050
6051         to_intel_atomic_state(state)->cdclk =
6052                 valleyview_calc_cdclk(dev_priv, max_pixclk);
6053
6054         return 0;
6055 }
6056
6057 static int broxton_modeset_calc_cdclk(struct drm_atomic_state *state)
6058 {
6059         struct drm_device *dev = state->dev;
6060         struct drm_i915_private *dev_priv = dev->dev_private;
6061         int max_pixclk = intel_mode_max_pixclk(dev, state);
6062
6063         if (max_pixclk < 0)
6064                 return max_pixclk;
6065
6066         to_intel_atomic_state(state)->cdclk =
6067                 broxton_calc_cdclk(dev_priv, max_pixclk);
6068
6069         return 0;
6070 }
6071
6072 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
6073 {
6074         unsigned int credits, default_credits;
6075
6076         if (IS_CHERRYVIEW(dev_priv))
6077                 default_credits = PFI_CREDIT(12);
6078         else
6079                 default_credits = PFI_CREDIT(8);
6080
6081         if (dev_priv->cdclk_freq >= dev_priv->czclk_freq) {
6082                 /* CHV suggested value is 31 or 63 */
6083                 if (IS_CHERRYVIEW(dev_priv))
6084                         credits = PFI_CREDIT_63;
6085                 else
6086                         credits = PFI_CREDIT(15);
6087         } else {
6088                 credits = default_credits;
6089         }
6090
6091         /*
6092          * WA - write default credits before re-programming
6093          * FIXME: should we also set the resend bit here?
6094          */
6095         I915_WRITE(GCI_CONTROL, VGA_FAST_MODE_DISABLE |
6096                    default_credits);
6097
6098         I915_WRITE(GCI_CONTROL, VGA_FAST_MODE_DISABLE |
6099                    credits | PFI_CREDIT_RESEND);
6100
6101         /*
6102          * FIXME is this guaranteed to clear
6103          * immediately or should we poll for it?
6104          */
6105         WARN_ON(I915_READ(GCI_CONTROL) & PFI_CREDIT_RESEND);
6106 }
6107
6108 static void valleyview_modeset_commit_cdclk(struct drm_atomic_state *old_state)
6109 {
6110         struct drm_device *dev = old_state->dev;
6111         unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
6112         struct drm_i915_private *dev_priv = dev->dev_private;
6113
6114         /*
6115          * FIXME: We can end up here with all power domains off, yet
6116          * with a CDCLK frequency other than the minimum. To account
6117          * for this take the PIPE-A power domain, which covers the HW
6118          * blocks needed for the following programming. This can be
6119          * removed once it's guaranteed that we get here either with
6120          * the minimum CDCLK set, or the required power domains
6121          * enabled.
6122          */
6123         intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
6124
6125         if (IS_CHERRYVIEW(dev))
6126                 cherryview_set_cdclk(dev, req_cdclk);
6127         else
6128                 valleyview_set_cdclk(dev, req_cdclk);
6129
6130         vlv_program_pfi_credits(dev_priv);
6131
6132         intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A);
6133 }
6134
6135 static void valleyview_crtc_enable(struct drm_crtc *crtc)
6136 {
6137         struct drm_device *dev = crtc->dev;
6138         struct drm_i915_private *dev_priv = to_i915(dev);
6139         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6140         struct intel_encoder *encoder;
6141         int pipe = intel_crtc->pipe;
6142         bool is_dsi;
6143
6144         if (WARN_ON(intel_crtc->active))
6145                 return;
6146
6147         is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
6148
6149         if (intel_crtc->config->has_dp_encoder)
6150                 intel_dp_set_m_n(intel_crtc, M1_N1);
6151
6152         intel_set_pipe_timings(intel_crtc);
6153
6154         if (IS_CHERRYVIEW(dev) && pipe == PIPE_B) {
6155                 struct drm_i915_private *dev_priv = dev->dev_private;
6156
6157                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
6158                 I915_WRITE(CHV_CANVAS(pipe), 0);
6159         }
6160
6161         i9xx_set_pipeconf(intel_crtc);
6162
6163         intel_crtc->active = true;
6164
6165         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6166
6167         for_each_encoder_on_crtc(dev, crtc, encoder)
6168                 if (encoder->pre_pll_enable)
6169                         encoder->pre_pll_enable(encoder);
6170
6171         if (!is_dsi) {
6172                 if (IS_CHERRYVIEW(dev)) {
6173                         chv_prepare_pll(intel_crtc, intel_crtc->config);
6174                         chv_enable_pll(intel_crtc, intel_crtc->config);
6175                 } else {
6176                         vlv_prepare_pll(intel_crtc, intel_crtc->config);
6177                         vlv_enable_pll(intel_crtc, intel_crtc->config);
6178                 }
6179         }
6180
6181         for_each_encoder_on_crtc(dev, crtc, encoder)
6182                 if (encoder->pre_enable)
6183                         encoder->pre_enable(encoder);
6184
6185         i9xx_pfit_enable(intel_crtc);
6186
6187         intel_crtc_load_lut(crtc);
6188
6189         intel_enable_pipe(intel_crtc);
6190
6191         assert_vblank_disabled(crtc);
6192         drm_crtc_vblank_on(crtc);
6193
6194         for_each_encoder_on_crtc(dev, crtc, encoder)
6195                 encoder->enable(encoder);
6196 }
6197
6198 static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
6199 {
6200         struct drm_device *dev = crtc->base.dev;
6201         struct drm_i915_private *dev_priv = dev->dev_private;
6202
6203         I915_WRITE(FP0(crtc->pipe), crtc->config->dpll_hw_state.fp0);
6204         I915_WRITE(FP1(crtc->pipe), crtc->config->dpll_hw_state.fp1);
6205 }
6206
6207 static void i9xx_crtc_enable(struct drm_crtc *crtc)
6208 {
6209         struct drm_device *dev = crtc->dev;
6210         struct drm_i915_private *dev_priv = to_i915(dev);
6211         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6212         struct intel_encoder *encoder;
6213         int pipe = intel_crtc->pipe;
6214
6215         if (WARN_ON(intel_crtc->active))
6216                 return;
6217
6218         i9xx_set_pll_dividers(intel_crtc);
6219
6220         if (intel_crtc->config->has_dp_encoder)
6221                 intel_dp_set_m_n(intel_crtc, M1_N1);
6222
6223         intel_set_pipe_timings(intel_crtc);
6224
6225         i9xx_set_pipeconf(intel_crtc);
6226
6227         intel_crtc->active = true;
6228
6229         if (!IS_GEN2(dev))
6230                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6231
6232         for_each_encoder_on_crtc(dev, crtc, encoder)
6233                 if (encoder->pre_enable)
6234                         encoder->pre_enable(encoder);
6235
6236         i9xx_enable_pll(intel_crtc);
6237
6238         i9xx_pfit_enable(intel_crtc);
6239
6240         intel_crtc_load_lut(crtc);
6241
6242         intel_update_watermarks(crtc);
6243         intel_enable_pipe(intel_crtc);
6244
6245         assert_vblank_disabled(crtc);
6246         drm_crtc_vblank_on(crtc);
6247
6248         for_each_encoder_on_crtc(dev, crtc, encoder)
6249                 encoder->enable(encoder);
6250 }
6251
6252 static void i9xx_pfit_disable(struct intel_crtc *crtc)
6253 {
6254         struct drm_device *dev = crtc->base.dev;
6255         struct drm_i915_private *dev_priv = dev->dev_private;
6256
6257         if (!crtc->config->gmch_pfit.control)
6258                 return;
6259
6260         assert_pipe_disabled(dev_priv, crtc->pipe);
6261
6262         DRM_DEBUG_DRIVER("disabling pfit, current: 0x%08x\n",
6263                          I915_READ(PFIT_CONTROL));
6264         I915_WRITE(PFIT_CONTROL, 0);
6265 }
6266
6267 static void i9xx_crtc_disable(struct drm_crtc *crtc)
6268 {
6269         struct drm_device *dev = crtc->dev;
6270         struct drm_i915_private *dev_priv = dev->dev_private;
6271         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6272         struct intel_encoder *encoder;
6273         int pipe = intel_crtc->pipe;
6274
6275         /*
6276          * On gen2 planes are double buffered but the pipe isn't, so we must
6277          * wait for planes to fully turn off before disabling the pipe.
6278          * We also need to wait on all gmch platforms because of the
6279          * self-refresh mode constraint explained above.
6280          */
6281         intel_wait_for_vblank(dev, pipe);
6282
6283         for_each_encoder_on_crtc(dev, crtc, encoder)
6284                 encoder->disable(encoder);
6285
6286         drm_crtc_vblank_off(crtc);
6287         assert_vblank_disabled(crtc);
6288
6289         intel_disable_pipe(intel_crtc);
6290
6291         i9xx_pfit_disable(intel_crtc);
6292
6293         for_each_encoder_on_crtc(dev, crtc, encoder)
6294                 if (encoder->post_disable)
6295                         encoder->post_disable(encoder);
6296
6297         if (!intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI)) {
6298                 if (IS_CHERRYVIEW(dev))
6299                         chv_disable_pll(dev_priv, pipe);
6300                 else if (IS_VALLEYVIEW(dev))
6301                         vlv_disable_pll(dev_priv, pipe);
6302                 else
6303                         i9xx_disable_pll(intel_crtc);
6304         }
6305
6306         for_each_encoder_on_crtc(dev, crtc, encoder)
6307                 if (encoder->post_pll_disable)
6308                         encoder->post_pll_disable(encoder);
6309
6310         if (!IS_GEN2(dev))
6311                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6312 }
6313
6314 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
6315 {
6316         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6317         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6318         enum intel_display_power_domain domain;
6319         unsigned long domains;
6320
6321         if (!intel_crtc->active)
6322                 return;
6323
6324         if (to_intel_plane_state(crtc->primary->state)->visible) {
6325                 intel_crtc_wait_for_pending_flips(crtc);
6326                 intel_pre_disable_primary(crtc);
6327
6328                 intel_crtc_disable_planes(crtc, 1 << drm_plane_index(crtc->primary));
6329                 to_intel_plane_state(crtc->primary->state)->visible = false;
6330         }
6331
6332         dev_priv->display.crtc_disable(crtc);
6333         intel_crtc->active = false;
6334         intel_update_watermarks(crtc);
6335         intel_disable_shared_dpll(intel_crtc);
6336
6337         domains = intel_crtc->enabled_power_domains;
6338         for_each_power_domain(domain, domains)
6339                 intel_display_power_put(dev_priv, domain);
6340         intel_crtc->enabled_power_domains = 0;
6341 }
6342
6343 /*
6344  * turn all crtc's off, but do not adjust state
6345  * This has to be paired with a call to intel_modeset_setup_hw_state.
6346  */
6347 int intel_display_suspend(struct drm_device *dev)
6348 {
6349         struct drm_mode_config *config = &dev->mode_config;
6350         struct drm_modeset_acquire_ctx *ctx = config->acquire_ctx;
6351         struct drm_atomic_state *state;
6352         struct drm_crtc *crtc;
6353         unsigned crtc_mask = 0;
6354         int ret = 0;
6355
6356         if (WARN_ON(!ctx))
6357                 return 0;
6358
6359         lockdep_assert_held(&ctx->ww_ctx);
6360         state = drm_atomic_state_alloc(dev);
6361         if (WARN_ON(!state))
6362                 return -ENOMEM;
6363
6364         state->acquire_ctx = ctx;
6365         state->allow_modeset = true;
6366
6367         for_each_crtc(dev, crtc) {
6368                 struct drm_crtc_state *crtc_state =
6369                         drm_atomic_get_crtc_state(state, crtc);
6370
6371                 ret = PTR_ERR_OR_ZERO(crtc_state);
6372                 if (ret)
6373                         goto free;
6374
6375                 if (!crtc_state->active)
6376                         continue;
6377
6378                 crtc_state->active = false;
6379                 crtc_mask |= 1 << drm_crtc_index(crtc);
6380         }
6381
6382         if (crtc_mask) {
6383                 ret = drm_atomic_commit(state);
6384
6385                 if (!ret) {
6386                         for_each_crtc(dev, crtc)
6387                                 if (crtc_mask & (1 << drm_crtc_index(crtc)))
6388                                         crtc->state->active = true;
6389
6390                         return ret;
6391                 }
6392         }
6393
6394 free:
6395         if (ret)
6396                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
6397         drm_atomic_state_free(state);
6398         return ret;
6399 }
6400
6401 void intel_encoder_destroy(struct drm_encoder *encoder)
6402 {
6403         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6404
6405         drm_encoder_cleanup(encoder);
6406         kfree(intel_encoder);
6407 }
6408
6409 /* Cross check the actual hw state with our own modeset state tracking (and it's
6410  * internal consistency). */
6411 static void intel_connector_check_state(struct intel_connector *connector)
6412 {
6413         struct drm_crtc *crtc = connector->base.state->crtc;
6414
6415         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
6416                       connector->base.base.id,
6417                       connector->base.name);
6418
6419         if (connector->get_hw_state(connector)) {
6420                 struct intel_encoder *encoder = connector->encoder;
6421                 struct drm_connector_state *conn_state = connector->base.state;
6422
6423                 I915_STATE_WARN(!crtc,
6424                          "connector enabled without attached crtc\n");
6425
6426                 if (!crtc)
6427                         return;
6428
6429                 I915_STATE_WARN(!crtc->state->active,
6430                       "connector is active, but attached crtc isn't\n");
6431
6432                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
6433                         return;
6434
6435                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
6436                         "atomic encoder doesn't match attached encoder\n");
6437
6438                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
6439                         "attached encoder crtc differs from connector crtc\n");
6440         } else {
6441                 I915_STATE_WARN(crtc && crtc->state->active,
6442                         "attached crtc is active, but connector isn't\n");
6443                 I915_STATE_WARN(!crtc && connector->base.state->best_encoder,
6444                         "best encoder set without crtc!\n");
6445         }
6446 }
6447
6448 int intel_connector_init(struct intel_connector *connector)
6449 {
6450         struct drm_connector_state *connector_state;
6451
6452         connector_state = kzalloc(sizeof *connector_state, GFP_KERNEL);
6453         if (!connector_state)
6454                 return -ENOMEM;
6455
6456         connector->base.state = connector_state;
6457         return 0;
6458 }
6459
6460 struct intel_connector *intel_connector_alloc(void)
6461 {
6462         struct intel_connector *connector;
6463
6464         connector = kzalloc(sizeof *connector, GFP_KERNEL);
6465         if (!connector)
6466                 return NULL;
6467
6468         if (intel_connector_init(connector) < 0) {
6469                 kfree(connector);
6470                 return NULL;
6471         }
6472
6473         return connector;
6474 }
6475
6476 /* Simple connector->get_hw_state implementation for encoders that support only
6477  * one connector and no cloning and hence the encoder state determines the state
6478  * of the connector. */
6479 bool intel_connector_get_hw_state(struct intel_connector *connector)
6480 {
6481         enum pipe pipe = 0;
6482         struct intel_encoder *encoder = connector->encoder;
6483
6484         return encoder->get_hw_state(encoder, &pipe);
6485 }
6486
6487 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
6488 {
6489         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
6490                 return crtc_state->fdi_lanes;
6491
6492         return 0;
6493 }
6494
6495 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
6496                                      struct intel_crtc_state *pipe_config)
6497 {
6498         struct drm_atomic_state *state = pipe_config->base.state;
6499         struct intel_crtc *other_crtc;
6500         struct intel_crtc_state *other_crtc_state;
6501
6502         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
6503                       pipe_name(pipe), pipe_config->fdi_lanes);
6504         if (pipe_config->fdi_lanes > 4) {
6505                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
6506                               pipe_name(pipe), pipe_config->fdi_lanes);
6507                 return -EINVAL;
6508         }
6509
6510         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
6511                 if (pipe_config->fdi_lanes > 2) {
6512                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
6513                                       pipe_config->fdi_lanes);
6514                         return -EINVAL;
6515                 } else {
6516                         return 0;
6517                 }
6518         }
6519
6520         if (INTEL_INFO(dev)->num_pipes == 2)
6521                 return 0;
6522
6523         /* Ivybridge 3 pipe is really complicated */
6524         switch (pipe) {
6525         case PIPE_A:
6526                 return 0;
6527         case PIPE_B:
6528                 if (pipe_config->fdi_lanes <= 2)
6529                         return 0;
6530
6531                 other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_C));
6532                 other_crtc_state =
6533                         intel_atomic_get_crtc_state(state, other_crtc);
6534                 if (IS_ERR(other_crtc_state))
6535                         return PTR_ERR(other_crtc_state);
6536
6537                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
6538                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
6539                                       pipe_name(pipe), pipe_config->fdi_lanes);
6540                         return -EINVAL;
6541                 }
6542                 return 0;
6543         case PIPE_C:
6544                 if (pipe_config->fdi_lanes > 2) {
6545                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
6546                                       pipe_name(pipe), pipe_config->fdi_lanes);
6547                         return -EINVAL;
6548                 }
6549
6550                 other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_B));
6551                 other_crtc_state =
6552                         intel_atomic_get_crtc_state(state, other_crtc);
6553                 if (IS_ERR(other_crtc_state))
6554                         return PTR_ERR(other_crtc_state);
6555
6556                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
6557                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
6558                         return -EINVAL;
6559                 }
6560                 return 0;
6561         default:
6562                 BUG();
6563         }
6564 }
6565
6566 #define RETRY 1
6567 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
6568                                        struct intel_crtc_state *pipe_config)
6569 {
6570         struct drm_device *dev = intel_crtc->base.dev;
6571         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6572         int lane, link_bw, fdi_dotclock, ret;
6573         bool needs_recompute = false;
6574
6575 retry:
6576         /* FDI is a binary signal running at ~2.7GHz, encoding
6577          * each output octet as 10 bits. The actual frequency
6578          * is stored as a divider into a 100MHz clock, and the
6579          * mode pixel clock is stored in units of 1KHz.
6580          * Hence the bw of each lane in terms of the mode signal
6581          * is:
6582          */
6583         link_bw = intel_fdi_link_freq(dev) * MHz(100)/KHz(1)/10;
6584
6585         fdi_dotclock = adjusted_mode->crtc_clock;
6586
6587         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
6588                                            pipe_config->pipe_bpp);
6589
6590         pipe_config->fdi_lanes = lane;
6591
6592         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
6593                                link_bw, &pipe_config->fdi_m_n);
6594
6595         ret = ironlake_check_fdi_lanes(intel_crtc->base.dev,
6596                                        intel_crtc->pipe, pipe_config);
6597         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
6598                 pipe_config->pipe_bpp -= 2*3;
6599                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
6600                               pipe_config->pipe_bpp);
6601                 needs_recompute = true;
6602                 pipe_config->bw_constrained = true;
6603
6604                 goto retry;
6605         }
6606
6607         if (needs_recompute)
6608                 return RETRY;
6609
6610         return ret;
6611 }
6612
6613 static bool pipe_config_supports_ips(struct drm_i915_private *dev_priv,
6614                                      struct intel_crtc_state *pipe_config)
6615 {
6616         if (pipe_config->pipe_bpp > 24)
6617                 return false;
6618
6619         /* HSW can handle pixel rate up to cdclk? */
6620         if (IS_HASWELL(dev_priv->dev))
6621                 return true;
6622
6623         /*
6624          * We compare against max which means we must take
6625          * the increased cdclk requirement into account when
6626          * calculating the new cdclk.
6627          *
6628          * Should measure whether using a lower cdclk w/o IPS
6629          */
6630         return ilk_pipe_pixel_rate(pipe_config) <=
6631                 dev_priv->max_cdclk_freq * 95 / 100;
6632 }
6633
6634 static void hsw_compute_ips_config(struct intel_crtc *crtc,
6635                                    struct intel_crtc_state *pipe_config)
6636 {
6637         struct drm_device *dev = crtc->base.dev;
6638         struct drm_i915_private *dev_priv = dev->dev_private;
6639
6640         pipe_config->ips_enabled = i915.enable_ips &&
6641                 hsw_crtc_supports_ips(crtc) &&
6642                 pipe_config_supports_ips(dev_priv, pipe_config);
6643 }
6644
6645 static int intel_crtc_compute_config(struct intel_crtc *crtc,
6646                                      struct intel_crtc_state *pipe_config)
6647 {
6648         struct drm_device *dev = crtc->base.dev;
6649         struct drm_i915_private *dev_priv = dev->dev_private;
6650         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6651
6652         /* FIXME should check pixel clock limits on all platforms */
6653         if (INTEL_INFO(dev)->gen < 4) {
6654                 int clock_limit = dev_priv->max_cdclk_freq;
6655
6656                 /*
6657                  * Enable pixel doubling when the dot clock
6658                  * is > 90% of the (display) core speed.
6659                  *
6660                  * GDG double wide on either pipe,
6661                  * otherwise pipe A only.
6662                  */
6663                 if ((crtc->pipe == PIPE_A || IS_I915G(dev)) &&
6664                     adjusted_mode->crtc_clock > clock_limit * 9 / 10) {
6665                         clock_limit *= 2;
6666                         pipe_config->double_wide = true;
6667                 }
6668
6669                 if (adjusted_mode->crtc_clock > clock_limit * 9 / 10)
6670                         return -EINVAL;
6671         }
6672
6673         /*
6674          * Pipe horizontal size must be even in:
6675          * - DVO ganged mode
6676          * - LVDS dual channel mode
6677          * - Double wide pipe
6678          */
6679         if ((intel_pipe_will_have_type(pipe_config, INTEL_OUTPUT_LVDS) &&
6680              intel_is_dual_link_lvds(dev)) || pipe_config->double_wide)
6681                 pipe_config->pipe_src_w &= ~1;
6682
6683         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
6684          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
6685          */
6686         if ((INTEL_INFO(dev)->gen > 4 || IS_G4X(dev)) &&
6687                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
6688                 return -EINVAL;
6689
6690         if (HAS_IPS(dev))
6691                 hsw_compute_ips_config(crtc, pipe_config);
6692
6693         if (pipe_config->has_pch_encoder)
6694                 return ironlake_fdi_compute_config(crtc, pipe_config);
6695
6696         return 0;
6697 }
6698
6699 static int skylake_get_display_clock_speed(struct drm_device *dev)
6700 {
6701         struct drm_i915_private *dev_priv = to_i915(dev);
6702         uint32_t lcpll1 = I915_READ(LCPLL1_CTL);
6703         uint32_t cdctl = I915_READ(CDCLK_CTL);
6704         uint32_t linkrate;
6705
6706         if (!(lcpll1 & LCPLL_PLL_ENABLE))
6707                 return 24000; /* 24MHz is the cd freq with NSSC ref */
6708
6709         if ((cdctl & CDCLK_FREQ_SEL_MASK) == CDCLK_FREQ_540)
6710                 return 540000;
6711
6712         linkrate = (I915_READ(DPLL_CTRL1) &
6713                     DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) >> 1;
6714
6715         if (linkrate == DPLL_CTRL1_LINK_RATE_2160 ||
6716             linkrate == DPLL_CTRL1_LINK_RATE_1080) {
6717                 /* vco 8640 */
6718                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
6719                 case CDCLK_FREQ_450_432:
6720                         return 432000;
6721                 case CDCLK_FREQ_337_308:
6722                         return 308570;
6723                 case CDCLK_FREQ_675_617:
6724                         return 617140;
6725                 default:
6726                         WARN(1, "Unknown cd freq selection\n");
6727                 }
6728         } else {
6729                 /* vco 8100 */
6730                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
6731                 case CDCLK_FREQ_450_432:
6732                         return 450000;
6733                 case CDCLK_FREQ_337_308:
6734                         return 337500;
6735                 case CDCLK_FREQ_675_617:
6736                         return 675000;
6737                 default:
6738                         WARN(1, "Unknown cd freq selection\n");
6739                 }
6740         }
6741
6742         /* error case, do as if DPLL0 isn't enabled */
6743         return 24000;
6744 }
6745
6746 static int broxton_get_display_clock_speed(struct drm_device *dev)
6747 {
6748         struct drm_i915_private *dev_priv = to_i915(dev);
6749         uint32_t cdctl = I915_READ(CDCLK_CTL);
6750         uint32_t pll_ratio = I915_READ(BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
6751         uint32_t pll_enab = I915_READ(BXT_DE_PLL_ENABLE);
6752         int cdclk;
6753
6754         if (!(pll_enab & BXT_DE_PLL_PLL_ENABLE))
6755                 return 19200;
6756
6757         cdclk = 19200 * pll_ratio / 2;
6758
6759         switch (cdctl & BXT_CDCLK_CD2X_DIV_SEL_MASK) {
6760         case BXT_CDCLK_CD2X_DIV_SEL_1:
6761                 return cdclk;  /* 576MHz or 624MHz */
6762         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
6763                 return cdclk * 2 / 3; /* 384MHz */
6764         case BXT_CDCLK_CD2X_DIV_SEL_2:
6765                 return cdclk / 2; /* 288MHz */
6766         case BXT_CDCLK_CD2X_DIV_SEL_4:
6767                 return cdclk / 4; /* 144MHz */
6768         }
6769
6770         /* error case, do as if DE PLL isn't enabled */
6771         return 19200;
6772 }
6773
6774 static int broadwell_get_display_clock_speed(struct drm_device *dev)
6775 {
6776         struct drm_i915_private *dev_priv = dev->dev_private;
6777         uint32_t lcpll = I915_READ(LCPLL_CTL);
6778         uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
6779
6780         if (lcpll & LCPLL_CD_SOURCE_FCLK)
6781                 return 800000;
6782         else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
6783                 return 450000;
6784         else if (freq == LCPLL_CLK_FREQ_450)
6785                 return 450000;
6786         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
6787                 return 540000;
6788         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
6789                 return 337500;
6790         else
6791                 return 675000;
6792 }
6793
6794 static int haswell_get_display_clock_speed(struct drm_device *dev)
6795 {
6796         struct drm_i915_private *dev_priv = dev->dev_private;
6797         uint32_t lcpll = I915_READ(LCPLL_CTL);
6798         uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
6799
6800         if (lcpll & LCPLL_CD_SOURCE_FCLK)
6801                 return 800000;
6802         else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
6803                 return 450000;
6804         else if (freq == LCPLL_CLK_FREQ_450)
6805                 return 450000;
6806         else if (IS_HSW_ULT(dev))
6807                 return 337500;
6808         else
6809                 return 540000;
6810 }
6811
6812 static int valleyview_get_display_clock_speed(struct drm_device *dev)
6813 {
6814         return vlv_get_cck_clock_hpll(to_i915(dev), "cdclk",
6815                                       CCK_DISPLAY_CLOCK_CONTROL);
6816 }
6817
6818 static int ilk_get_display_clock_speed(struct drm_device *dev)
6819 {
6820         return 450000;
6821 }
6822
6823 static int i945_get_display_clock_speed(struct drm_device *dev)
6824 {
6825         return 400000;
6826 }
6827
6828 static int i915_get_display_clock_speed(struct drm_device *dev)
6829 {
6830         return 333333;
6831 }
6832
6833 static int i9xx_misc_get_display_clock_speed(struct drm_device *dev)
6834 {
6835         return 200000;
6836 }
6837
6838 static int pnv_get_display_clock_speed(struct drm_device *dev)
6839 {
6840         u16 gcfgc = 0;
6841
6842         pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
6843
6844         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
6845         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
6846                 return 266667;
6847         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
6848                 return 333333;
6849         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
6850                 return 444444;
6851         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
6852                 return 200000;
6853         default:
6854                 DRM_ERROR("Unknown pnv display core clock 0x%04x\n", gcfgc);
6855         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
6856                 return 133333;
6857         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
6858                 return 166667;
6859         }
6860 }
6861
6862 static int i915gm_get_display_clock_speed(struct drm_device *dev)
6863 {
6864         u16 gcfgc = 0;
6865
6866         pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
6867
6868         if (gcfgc & GC_LOW_FREQUENCY_ENABLE)
6869                 return 133333;
6870         else {
6871                 switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
6872                 case GC_DISPLAY_CLOCK_333_MHZ:
6873                         return 333333;
6874                 default:
6875                 case GC_DISPLAY_CLOCK_190_200_MHZ:
6876                         return 190000;
6877                 }
6878         }
6879 }
6880
6881 static int i865_get_display_clock_speed(struct drm_device *dev)
6882 {
6883         return 266667;
6884 }
6885
6886 static int i85x_get_display_clock_speed(struct drm_device *dev)
6887 {
6888         u16 hpllcc = 0;
6889
6890         /*
6891          * 852GM/852GMV only supports 133 MHz and the HPLLCC
6892          * encoding is different :(
6893          * FIXME is this the right way to detect 852GM/852GMV?
6894          */
6895         if (dev->pdev->revision == 0x1)
6896                 return 133333;
6897
6898         pci_bus_read_config_word(dev->pdev->bus,
6899                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
6900
6901         /* Assume that the hardware is in the high speed state.  This
6902          * should be the default.
6903          */
6904         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
6905         case GC_CLOCK_133_200:
6906         case GC_CLOCK_133_200_2:
6907         case GC_CLOCK_100_200:
6908                 return 200000;
6909         case GC_CLOCK_166_250:
6910                 return 250000;
6911         case GC_CLOCK_100_133:
6912                 return 133333;
6913         case GC_CLOCK_133_266:
6914         case GC_CLOCK_133_266_2:
6915         case GC_CLOCK_166_266:
6916                 return 266667;
6917         }
6918
6919         /* Shouldn't happen */
6920         return 0;
6921 }
6922
6923 static int i830_get_display_clock_speed(struct drm_device *dev)
6924 {
6925         return 133333;
6926 }
6927
6928 static unsigned int intel_hpll_vco(struct drm_device *dev)
6929 {
6930         struct drm_i915_private *dev_priv = dev->dev_private;
6931         static const unsigned int blb_vco[8] = {
6932                 [0] = 3200000,
6933                 [1] = 4000000,
6934                 [2] = 5333333,
6935                 [3] = 4800000,
6936                 [4] = 6400000,
6937         };
6938         static const unsigned int pnv_vco[8] = {
6939                 [0] = 3200000,
6940                 [1] = 4000000,
6941                 [2] = 5333333,
6942                 [3] = 4800000,
6943                 [4] = 2666667,
6944         };
6945         static const unsigned int cl_vco[8] = {
6946                 [0] = 3200000,
6947                 [1] = 4000000,
6948                 [2] = 5333333,
6949                 [3] = 6400000,
6950                 [4] = 3333333,
6951                 [5] = 3566667,
6952                 [6] = 4266667,
6953         };
6954         static const unsigned int elk_vco[8] = {
6955                 [0] = 3200000,
6956                 [1] = 4000000,
6957                 [2] = 5333333,
6958                 [3] = 4800000,
6959         };
6960         static const unsigned int ctg_vco[8] = {
6961                 [0] = 3200000,
6962                 [1] = 4000000,
6963                 [2] = 5333333,
6964                 [3] = 6400000,
6965                 [4] = 2666667,
6966                 [5] = 4266667,
6967         };
6968         const unsigned int *vco_table;
6969         unsigned int vco;
6970         uint8_t tmp = 0;
6971
6972         /* FIXME other chipsets? */
6973         if (IS_GM45(dev))
6974                 vco_table = ctg_vco;
6975         else if (IS_G4X(dev))
6976                 vco_table = elk_vco;
6977         else if (IS_CRESTLINE(dev))
6978                 vco_table = cl_vco;
6979         else if (IS_PINEVIEW(dev))
6980                 vco_table = pnv_vco;
6981         else if (IS_G33(dev))
6982                 vco_table = blb_vco;
6983         else
6984                 return 0;
6985
6986         tmp = I915_READ(IS_MOBILE(dev) ? HPLLVCO_MOBILE : HPLLVCO);
6987
6988         vco = vco_table[tmp & 0x7];
6989         if (vco == 0)
6990                 DRM_ERROR("Bad HPLL VCO (HPLLVCO=0x%02x)\n", tmp);
6991         else
6992                 DRM_DEBUG_KMS("HPLL VCO %u kHz\n", vco);
6993
6994         return vco;
6995 }
6996
6997 static int gm45_get_display_clock_speed(struct drm_device *dev)
6998 {
6999         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
7000         uint16_t tmp = 0;
7001
7002         pci_read_config_word(dev->pdev, GCFGC, &tmp);
7003
7004         cdclk_sel = (tmp >> 12) & 0x1;
7005
7006         switch (vco) {
7007         case 2666667:
7008         case 4000000:
7009         case 5333333:
7010                 return cdclk_sel ? 333333 : 222222;
7011         case 3200000:
7012                 return cdclk_sel ? 320000 : 228571;
7013         default:
7014                 DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n", vco, tmp);
7015                 return 222222;
7016         }
7017 }
7018
7019 static int i965gm_get_display_clock_speed(struct drm_device *dev)
7020 {
7021         static const uint8_t div_3200[] = { 16, 10,  8 };
7022         static const uint8_t div_4000[] = { 20, 12, 10 };
7023         static const uint8_t div_5333[] = { 24, 16, 14 };
7024         const uint8_t *div_table;
7025         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
7026         uint16_t tmp = 0;
7027
7028         pci_read_config_word(dev->pdev, GCFGC, &tmp);
7029
7030         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
7031
7032         if (cdclk_sel >= ARRAY_SIZE(div_3200))
7033                 goto fail;
7034
7035         switch (vco) {
7036         case 3200000:
7037                 div_table = div_3200;
7038                 break;
7039         case 4000000:
7040                 div_table = div_4000;
7041                 break;
7042         case 5333333:
7043                 div_table = div_5333;
7044                 break;
7045         default:
7046                 goto fail;
7047         }
7048
7049         return DIV_ROUND_CLOSEST(vco, div_table[cdclk_sel]);
7050
7051 fail:
7052         DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n", vco, tmp);
7053         return 200000;
7054 }
7055
7056 static int g33_get_display_clock_speed(struct drm_device *dev)
7057 {
7058         static const uint8_t div_3200[] = { 12, 10,  8,  7, 5, 16 };
7059         static const uint8_t div_4000[] = { 14, 12, 10,  8, 6, 20 };
7060         static const uint8_t div_4800[] = { 20, 14, 12, 10, 8, 24 };
7061         static const uint8_t div_5333[] = { 20, 16, 12, 12, 8, 28 };
7062         const uint8_t *div_table;
7063         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
7064         uint16_t tmp = 0;
7065
7066         pci_read_config_word(dev->pdev, GCFGC, &tmp);
7067
7068         cdclk_sel = (tmp >> 4) & 0x7;
7069
7070         if (cdclk_sel >= ARRAY_SIZE(div_3200))
7071                 goto fail;
7072
7073         switch (vco) {
7074         case 3200000:
7075                 div_table = div_3200;
7076                 break;
7077         case 4000000:
7078                 div_table = div_4000;
7079                 break;
7080         case 4800000:
7081                 div_table = div_4800;
7082                 break;
7083         case 5333333:
7084                 div_table = div_5333;
7085                 break;
7086         default:
7087                 goto fail;
7088         }
7089
7090         return DIV_ROUND_CLOSEST(vco, div_table[cdclk_sel]);
7091
7092 fail:
7093         DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n", vco, tmp);
7094         return 190476;
7095 }
7096
7097 static void
7098 intel_reduce_m_n_ratio(uint32_t *num, uint32_t *den)
7099 {
7100         while (*num > DATA_LINK_M_N_MASK ||
7101                *den > DATA_LINK_M_N_MASK) {
7102                 *num >>= 1;
7103                 *den >>= 1;
7104         }
7105 }
7106
7107 static void compute_m_n(unsigned int m, unsigned int n,
7108                         uint32_t *ret_m, uint32_t *ret_n)
7109 {
7110         *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
7111         *ret_m = div_u64((uint64_t) m * *ret_n, n);
7112         intel_reduce_m_n_ratio(ret_m, ret_n);
7113 }
7114
7115 void
7116 intel_link_compute_m_n(int bits_per_pixel, int nlanes,
7117                        int pixel_clock, int link_clock,
7118                        struct intel_link_m_n *m_n)
7119 {
7120         m_n->tu = 64;
7121
7122         compute_m_n(bits_per_pixel * pixel_clock,
7123                     link_clock * nlanes * 8,
7124                     &m_n->gmch_m, &m_n->gmch_n);
7125
7126         compute_m_n(pixel_clock, link_clock,
7127                     &m_n->link_m, &m_n->link_n);
7128 }
7129
7130 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7131 {
7132         if (i915.panel_use_ssc >= 0)
7133                 return i915.panel_use_ssc != 0;
7134         return dev_priv->vbt.lvds_use_ssc
7135                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7136 }
7137
7138 static int i9xx_get_refclk(const struct intel_crtc_state *crtc_state,
7139                            int num_connectors)
7140 {
7141         struct drm_device *dev = crtc_state->base.crtc->dev;
7142         struct drm_i915_private *dev_priv = dev->dev_private;
7143         int refclk;
7144
7145         WARN_ON(!crtc_state->base.state);
7146
7147         if (IS_VALLEYVIEW(dev) || IS_BROXTON(dev)) {
7148                 refclk = 100000;
7149         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7150             intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
7151                 refclk = dev_priv->vbt.lvds_ssc_freq;
7152                 DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7153         } else if (!IS_GEN2(dev)) {
7154                 refclk = 96000;
7155         } else {
7156                 refclk = 48000;
7157         }
7158
7159         return refclk;
7160 }
7161
7162 static uint32_t pnv_dpll_compute_fp(struct dpll *dpll)
7163 {
7164         return (1 << dpll->n) << 16 | dpll->m2;
7165 }
7166
7167 static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll)
7168 {
7169         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7170 }
7171
7172 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7173                                      struct intel_crtc_state *crtc_state,
7174                                      intel_clock_t *reduced_clock)
7175 {
7176         struct drm_device *dev = crtc->base.dev;
7177         u32 fp, fp2 = 0;
7178
7179         if (IS_PINEVIEW(dev)) {
7180                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7181                 if (reduced_clock)
7182                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7183         } else {
7184                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7185                 if (reduced_clock)
7186                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7187         }
7188
7189         crtc_state->dpll_hw_state.fp0 = fp;
7190
7191         crtc->lowfreq_avail = false;
7192         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7193             reduced_clock) {
7194                 crtc_state->dpll_hw_state.fp1 = fp2;
7195                 crtc->lowfreq_avail = true;
7196         } else {
7197                 crtc_state->dpll_hw_state.fp1 = fp;
7198         }
7199 }
7200
7201 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7202                 pipe)
7203 {
7204         u32 reg_val;
7205
7206         /*
7207          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7208          * and set it to a reasonable value instead.
7209          */
7210         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7211         reg_val &= 0xffffff00;
7212         reg_val |= 0x00000030;
7213         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7214
7215         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7216         reg_val &= 0x8cffffff;
7217         reg_val = 0x8c000000;
7218         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7219
7220         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7221         reg_val &= 0xffffff00;
7222         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7223
7224         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7225         reg_val &= 0x00ffffff;
7226         reg_val |= 0xb0000000;
7227         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7228 }
7229
7230 static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
7231                                          struct intel_link_m_n *m_n)
7232 {
7233         struct drm_device *dev = crtc->base.dev;
7234         struct drm_i915_private *dev_priv = dev->dev_private;
7235         int pipe = crtc->pipe;
7236
7237         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7238         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7239         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7240         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7241 }
7242
7243 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
7244                                          struct intel_link_m_n *m_n,
7245                                          struct intel_link_m_n *m2_n2)
7246 {
7247         struct drm_device *dev = crtc->base.dev;
7248         struct drm_i915_private *dev_priv = dev->dev_private;
7249         int pipe = crtc->pipe;
7250         enum transcoder transcoder = crtc->config->cpu_transcoder;
7251
7252         if (INTEL_INFO(dev)->gen >= 5) {
7253                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7254                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7255                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7256                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7257                 /* M2_N2 registers to be set only for gen < 8 (M2_N2 available
7258                  * for gen < 8) and if DRRS is supported (to make sure the
7259                  * registers are not unnecessarily accessed).
7260                  */
7261                 if (m2_n2 && (IS_CHERRYVIEW(dev) || INTEL_INFO(dev)->gen < 8) &&
7262                         crtc->config->has_drrs) {
7263                         I915_WRITE(PIPE_DATA_M2(transcoder),
7264                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
7265                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
7266                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
7267                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
7268                 }
7269         } else {
7270                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7271                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
7272                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
7273                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
7274         }
7275 }
7276
7277 void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n)
7278 {
7279         struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
7280
7281         if (m_n == M1_N1) {
7282                 dp_m_n = &crtc->config->dp_m_n;
7283                 dp_m2_n2 = &crtc->config->dp_m2_n2;
7284         } else if (m_n == M2_N2) {
7285
7286                 /*
7287                  * M2_N2 registers are not supported. Hence m2_n2 divider value
7288                  * needs to be programmed into M1_N1.
7289                  */
7290                 dp_m_n = &crtc->config->dp_m2_n2;
7291         } else {
7292                 DRM_ERROR("Unsupported divider value\n");
7293                 return;
7294         }
7295
7296         if (crtc->config->has_pch_encoder)
7297                 intel_pch_transcoder_set_m_n(crtc, &crtc->config->dp_m_n);
7298         else
7299                 intel_cpu_transcoder_set_m_n(crtc, dp_m_n, dp_m2_n2);
7300 }
7301
7302 static void vlv_compute_dpll(struct intel_crtc *crtc,
7303                              struct intel_crtc_state *pipe_config)
7304 {
7305         u32 dpll, dpll_md;
7306
7307         /*
7308          * Enable DPIO clock input. We should never disable the reference
7309          * clock for pipe B, since VGA hotplug / manual detection depends
7310          * on it.
7311          */
7312         dpll = DPLL_EXT_BUFFER_ENABLE_VLV | DPLL_REF_CLK_ENABLE_VLV |
7313                 DPLL_VGA_MODE_DIS | DPLL_INTEGRATED_REF_CLK_VLV;
7314         /* We should never disable this, set it here for state tracking */
7315         if (crtc->pipe == PIPE_B)
7316                 dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7317         dpll |= DPLL_VCO_ENABLE;
7318         pipe_config->dpll_hw_state.dpll = dpll;
7319
7320         dpll_md = (pipe_config->pixel_multiplier - 1)
7321                 << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7322         pipe_config->dpll_hw_state.dpll_md = dpll_md;
7323 }
7324
7325 static void vlv_prepare_pll(struct intel_crtc *crtc,
7326                             const struct intel_crtc_state *pipe_config)
7327 {
7328         struct drm_device *dev = crtc->base.dev;
7329         struct drm_i915_private *dev_priv = dev->dev_private;
7330         int pipe = crtc->pipe;
7331         u32 mdiv;
7332         u32 bestn, bestm1, bestm2, bestp1, bestp2;
7333         u32 coreclk, reg_val;
7334
7335         mutex_lock(&dev_priv->sb_lock);
7336
7337         bestn = pipe_config->dpll.n;
7338         bestm1 = pipe_config->dpll.m1;
7339         bestm2 = pipe_config->dpll.m2;
7340         bestp1 = pipe_config->dpll.p1;
7341         bestp2 = pipe_config->dpll.p2;
7342
7343         /* See eDP HDMI DPIO driver vbios notes doc */
7344
7345         /* PLL B needs special handling */
7346         if (pipe == PIPE_B)
7347                 vlv_pllb_recal_opamp(dev_priv, pipe);
7348
7349         /* Set up Tx target for periodic Rcomp update */
7350         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7351
7352         /* Disable target IRef on PLL */
7353         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7354         reg_val &= 0x00ffffff;
7355         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7356
7357         /* Disable fast lock */
7358         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7359
7360         /* Set idtafcrecal before PLL is enabled */
7361         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7362         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7363         mdiv |= ((bestn << DPIO_N_SHIFT));
7364         mdiv |= (1 << DPIO_K_SHIFT);
7365
7366         /*
7367          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7368          * but we don't support that).
7369          * Note: don't use the DAC post divider as it seems unstable.
7370          */
7371         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7372         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7373
7374         mdiv |= DPIO_ENABLE_CALIBRATION;
7375         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7376
7377         /* Set HBR and RBR LPF coefficients */
7378         if (pipe_config->port_clock == 162000 ||
7379             intel_pipe_has_type(crtc, INTEL_OUTPUT_ANALOG) ||
7380             intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI))
7381                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7382                                  0x009f0003);
7383         else
7384                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7385                                  0x00d0000f);
7386
7387         if (pipe_config->has_dp_encoder) {
7388                 /* Use SSC source */
7389                 if (pipe == PIPE_A)
7390                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7391                                          0x0df40000);
7392                 else
7393                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7394                                          0x0df70000);
7395         } else { /* HDMI or VGA */
7396                 /* Use bend source */
7397                 if (pipe == PIPE_A)
7398                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7399                                          0x0df70000);
7400                 else
7401                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7402                                          0x0df40000);
7403         }
7404
7405         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7406         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7407         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
7408             intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))
7409                 coreclk |= 0x01000000;
7410         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7411
7412         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7413         mutex_unlock(&dev_priv->sb_lock);
7414 }
7415
7416 static void chv_compute_dpll(struct intel_crtc *crtc,
7417                              struct intel_crtc_state *pipe_config)
7418 {
7419         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
7420                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS |
7421                 DPLL_VCO_ENABLE;
7422         if (crtc->pipe != PIPE_A)
7423                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7424
7425         pipe_config->dpll_hw_state.dpll_md =
7426                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7427 }
7428
7429 static void chv_prepare_pll(struct intel_crtc *crtc,
7430                             const struct intel_crtc_state *pipe_config)
7431 {
7432         struct drm_device *dev = crtc->base.dev;
7433         struct drm_i915_private *dev_priv = dev->dev_private;
7434         int pipe = crtc->pipe;
7435         int dpll_reg = DPLL(crtc->pipe);
7436         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7437         u32 loopfilter, tribuf_calcntr;
7438         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7439         u32 dpio_val;
7440         int vco;
7441
7442         bestn = pipe_config->dpll.n;
7443         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7444         bestm1 = pipe_config->dpll.m1;
7445         bestm2 = pipe_config->dpll.m2 >> 22;
7446         bestp1 = pipe_config->dpll.p1;
7447         bestp2 = pipe_config->dpll.p2;
7448         vco = pipe_config->dpll.vco;
7449         dpio_val = 0;
7450         loopfilter = 0;
7451
7452         /*
7453          * Enable Refclk and SSC
7454          */
7455         I915_WRITE(dpll_reg,
7456                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7457
7458         mutex_lock(&dev_priv->sb_lock);
7459
7460         /* p1 and p2 divider */
7461         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7462                         5 << DPIO_CHV_S1_DIV_SHIFT |
7463                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7464                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7465                         1 << DPIO_CHV_K_DIV_SHIFT);
7466
7467         /* Feedback post-divider - m2 */
7468         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7469
7470         /* Feedback refclk divider - n and m1 */
7471         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7472                         DPIO_CHV_M1_DIV_BY_2 |
7473                         1 << DPIO_CHV_N_DIV_SHIFT);
7474
7475         /* M2 fraction division */
7476         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7477
7478         /* M2 fraction division enable */
7479         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7480         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7481         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7482         if (bestm2_frac)
7483                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7484         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7485
7486         /* Program digital lock detect threshold */
7487         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7488         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7489                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7490         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7491         if (!bestm2_frac)
7492                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7493         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7494
7495         /* Loop filter */
7496         if (vco == 5400000) {
7497                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7498                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7499                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7500                 tribuf_calcntr = 0x9;
7501         } else if (vco <= 6200000) {
7502                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7503                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7504                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7505                 tribuf_calcntr = 0x9;
7506         } else if (vco <= 6480000) {
7507                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7508                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7509                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7510                 tribuf_calcntr = 0x8;
7511         } else {
7512                 /* Not supported. Apply the same limits as in the max case */
7513                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7514                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7515                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7516                 tribuf_calcntr = 0;
7517         }
7518         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7519
7520         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7521         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7522         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7523         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7524
7525         /* AFC Recal */
7526         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7527                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7528                         DPIO_AFC_RECAL);
7529
7530         mutex_unlock(&dev_priv->sb_lock);
7531 }
7532
7533 /**
7534  * vlv_force_pll_on - forcibly enable just the PLL
7535  * @dev_priv: i915 private structure
7536  * @pipe: pipe PLL to enable
7537  * @dpll: PLL configuration
7538  *
7539  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7540  * in cases where we need the PLL enabled even when @pipe is not going to
7541  * be enabled.
7542  */
7543 void vlv_force_pll_on(struct drm_device *dev, enum pipe pipe,
7544                       const struct dpll *dpll)
7545 {
7546         struct intel_crtc *crtc =
7547                 to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
7548         struct intel_crtc_state pipe_config = {
7549                 .base.crtc = &crtc->base,
7550                 .pixel_multiplier = 1,
7551                 .dpll = *dpll,
7552         };
7553
7554         if (IS_CHERRYVIEW(dev)) {
7555                 chv_compute_dpll(crtc, &pipe_config);
7556                 chv_prepare_pll(crtc, &pipe_config);
7557                 chv_enable_pll(crtc, &pipe_config);
7558         } else {
7559                 vlv_compute_dpll(crtc, &pipe_config);
7560                 vlv_prepare_pll(crtc, &pipe_config);
7561                 vlv_enable_pll(crtc, &pipe_config);
7562         }
7563 }
7564
7565 /**
7566  * vlv_force_pll_off - forcibly disable just the PLL
7567  * @dev_priv: i915 private structure
7568  * @pipe: pipe PLL to disable
7569  *
7570  * Disable the PLL for @pipe. To be used in cases where we need
7571  * the PLL enabled even when @pipe is not going to be enabled.
7572  */
7573 void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe)
7574 {
7575         if (IS_CHERRYVIEW(dev))
7576                 chv_disable_pll(to_i915(dev), pipe);
7577         else
7578                 vlv_disable_pll(to_i915(dev), pipe);
7579 }
7580
7581 static void i9xx_compute_dpll(struct intel_crtc *crtc,
7582                               struct intel_crtc_state *crtc_state,
7583                               intel_clock_t *reduced_clock,
7584                               int num_connectors)
7585 {
7586         struct drm_device *dev = crtc->base.dev;
7587         struct drm_i915_private *dev_priv = dev->dev_private;
7588         u32 dpll;
7589         bool is_sdvo;
7590         struct dpll *clock = &crtc_state->dpll;
7591
7592         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7593
7594         is_sdvo = intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO) ||
7595                 intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI);
7596
7597         dpll = DPLL_VGA_MODE_DIS;
7598
7599         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
7600                 dpll |= DPLLB_MODE_LVDS;
7601         else
7602                 dpll |= DPLLB_MODE_DAC_SERIAL;
7603
7604         if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
7605                 dpll |= (crtc_state->pixel_multiplier - 1)
7606                         << SDVO_MULTIPLIER_SHIFT_HIRES;
7607         }
7608
7609         if (is_sdvo)
7610                 dpll |= DPLL_SDVO_HIGH_SPEED;
7611
7612         if (crtc_state->has_dp_encoder)
7613                 dpll |= DPLL_SDVO_HIGH_SPEED;
7614
7615         /* compute bitmask from p1 value */
7616         if (IS_PINEVIEW(dev))
7617                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
7618         else {
7619                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7620                 if (IS_G4X(dev) && reduced_clock)
7621                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
7622         }
7623         switch (clock->p2) {
7624         case 5:
7625                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
7626                 break;
7627         case 7:
7628                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
7629                 break;
7630         case 10:
7631                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
7632                 break;
7633         case 14:
7634                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
7635                 break;
7636         }
7637         if (INTEL_INFO(dev)->gen >= 4)
7638                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
7639
7640         if (crtc_state->sdvo_tv_clock)
7641                 dpll |= PLL_REF_INPUT_TVCLKINBC;
7642         else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7643                  intel_panel_use_ssc(dev_priv) && num_connectors < 2)
7644                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7645         else
7646                 dpll |= PLL_REF_INPUT_DREFCLK;
7647
7648         dpll |= DPLL_VCO_ENABLE;
7649         crtc_state->dpll_hw_state.dpll = dpll;
7650
7651         if (INTEL_INFO(dev)->gen >= 4) {
7652                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
7653                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7654                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
7655         }
7656 }
7657
7658 static void i8xx_compute_dpll(struct intel_crtc *crtc,
7659                               struct intel_crtc_state *crtc_state,
7660                               intel_clock_t *reduced_clock,
7661                               int num_connectors)
7662 {
7663         struct drm_device *dev = crtc->base.dev;
7664         struct drm_i915_private *dev_priv = dev->dev_private;
7665         u32 dpll;
7666         struct dpll *clock = &crtc_state->dpll;
7667
7668         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7669
7670         dpll = DPLL_VGA_MODE_DIS;
7671
7672         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7673                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7674         } else {
7675                 if (clock->p1 == 2)
7676                         dpll |= PLL_P1_DIVIDE_BY_TWO;
7677                 else
7678                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7679                 if (clock->p2 == 4)
7680                         dpll |= PLL_P2_DIVIDE_BY_4;
7681         }
7682
7683         if (!IS_I830(dev) && intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO))
7684                 dpll |= DPLL_DVO_2X_MODE;
7685
7686         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7687                  intel_panel_use_ssc(dev_priv) && num_connectors < 2)
7688                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7689         else
7690                 dpll |= PLL_REF_INPUT_DREFCLK;
7691
7692         dpll |= DPLL_VCO_ENABLE;
7693         crtc_state->dpll_hw_state.dpll = dpll;
7694 }
7695
7696 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
7697 {
7698         struct drm_device *dev = intel_crtc->base.dev;
7699         struct drm_i915_private *dev_priv = dev->dev_private;
7700         enum pipe pipe = intel_crtc->pipe;
7701         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
7702         const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
7703         uint32_t crtc_vtotal, crtc_vblank_end;
7704         int vsyncshift = 0;
7705
7706         /* We need to be careful not to changed the adjusted mode, for otherwise
7707          * the hw state checker will get angry at the mismatch. */
7708         crtc_vtotal = adjusted_mode->crtc_vtotal;
7709         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
7710
7711         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
7712                 /* the chip adds 2 halflines automatically */
7713                 crtc_vtotal -= 1;
7714                 crtc_vblank_end -= 1;
7715
7716                 if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
7717                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
7718                 else
7719                         vsyncshift = adjusted_mode->crtc_hsync_start -
7720                                 adjusted_mode->crtc_htotal / 2;
7721                 if (vsyncshift < 0)
7722                         vsyncshift += adjusted_mode->crtc_htotal;
7723         }
7724
7725         if (INTEL_INFO(dev)->gen > 3)
7726                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
7727
7728         I915_WRITE(HTOTAL(cpu_transcoder),
7729                    (adjusted_mode->crtc_hdisplay - 1) |
7730                    ((adjusted_mode->crtc_htotal - 1) << 16));
7731         I915_WRITE(HBLANK(cpu_transcoder),
7732                    (adjusted_mode->crtc_hblank_start - 1) |
7733                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
7734         I915_WRITE(HSYNC(cpu_transcoder),
7735                    (adjusted_mode->crtc_hsync_start - 1) |
7736                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
7737
7738         I915_WRITE(VTOTAL(cpu_transcoder),
7739                    (adjusted_mode->crtc_vdisplay - 1) |
7740                    ((crtc_vtotal - 1) << 16));
7741         I915_WRITE(VBLANK(cpu_transcoder),
7742                    (adjusted_mode->crtc_vblank_start - 1) |
7743                    ((crtc_vblank_end - 1) << 16));
7744         I915_WRITE(VSYNC(cpu_transcoder),
7745                    (adjusted_mode->crtc_vsync_start - 1) |
7746                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
7747
7748         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
7749          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
7750          * documented on the DDI_FUNC_CTL register description, EDP Input Select
7751          * bits. */
7752         if (IS_HASWELL(dev) && cpu_transcoder == TRANSCODER_EDP &&
7753             (pipe == PIPE_B || pipe == PIPE_C))
7754                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
7755
7756         /* pipesrc controls the size that is scaled from, which should
7757          * always be the user's requested size.
7758          */
7759         I915_WRITE(PIPESRC(pipe),
7760                    ((intel_crtc->config->pipe_src_w - 1) << 16) |
7761                    (intel_crtc->config->pipe_src_h - 1));
7762 }
7763
7764 static void intel_get_pipe_timings(struct intel_crtc *crtc,
7765                                    struct intel_crtc_state *pipe_config)
7766 {
7767         struct drm_device *dev = crtc->base.dev;
7768         struct drm_i915_private *dev_priv = dev->dev_private;
7769         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
7770         uint32_t tmp;
7771
7772         tmp = I915_READ(HTOTAL(cpu_transcoder));
7773         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
7774         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
7775         tmp = I915_READ(HBLANK(cpu_transcoder));
7776         pipe_config->base.adjusted_mode.crtc_hblank_start = (tmp & 0xffff) + 1;
7777         pipe_config->base.adjusted_mode.crtc_hblank_end = ((tmp >> 16) & 0xffff) + 1;
7778         tmp = I915_READ(HSYNC(cpu_transcoder));
7779         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
7780         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
7781
7782         tmp = I915_READ(VTOTAL(cpu_transcoder));
7783         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
7784         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
7785         tmp = I915_READ(VBLANK(cpu_transcoder));
7786         pipe_config->base.adjusted_mode.crtc_vblank_start = (tmp & 0xffff) + 1;
7787         pipe_config->base.adjusted_mode.crtc_vblank_end = ((tmp >> 16) & 0xffff) + 1;
7788         tmp = I915_READ(VSYNC(cpu_transcoder));
7789         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
7790         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
7791
7792         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
7793                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
7794                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
7795                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
7796         }
7797
7798         tmp = I915_READ(PIPESRC(crtc->pipe));
7799         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
7800         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
7801
7802         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
7803         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
7804 }
7805
7806 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
7807                                  struct intel_crtc_state *pipe_config)
7808 {
7809         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
7810         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
7811         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
7812         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
7813
7814         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
7815         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
7816         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
7817         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
7818
7819         mode->flags = pipe_config->base.adjusted_mode.flags;
7820         mode->type = DRM_MODE_TYPE_DRIVER;
7821
7822         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
7823         mode->flags |= pipe_config->base.adjusted_mode.flags;
7824
7825         mode->hsync = drm_mode_hsync(mode);
7826         mode->vrefresh = drm_mode_vrefresh(mode);
7827         drm_mode_set_name(mode);
7828 }
7829
7830 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
7831 {
7832         struct drm_device *dev = intel_crtc->base.dev;
7833         struct drm_i915_private *dev_priv = dev->dev_private;
7834         uint32_t pipeconf;
7835
7836         pipeconf = 0;
7837
7838         if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
7839             (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
7840                 pipeconf |= I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE;
7841
7842         if (intel_crtc->config->double_wide)
7843                 pipeconf |= PIPECONF_DOUBLE_WIDE;
7844
7845         /* only g4x and later have fancy bpc/dither controls */
7846         if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
7847                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
7848                 if (intel_crtc->config->dither && intel_crtc->config->pipe_bpp != 30)
7849                         pipeconf |= PIPECONF_DITHER_EN |
7850                                     PIPECONF_DITHER_TYPE_SP;
7851
7852                 switch (intel_crtc->config->pipe_bpp) {
7853                 case 18:
7854                         pipeconf |= PIPECONF_6BPC;
7855                         break;
7856                 case 24:
7857                         pipeconf |= PIPECONF_8BPC;
7858                         break;
7859                 case 30:
7860                         pipeconf |= PIPECONF_10BPC;
7861                         break;
7862                 default:
7863                         /* Case prevented by intel_choose_pipe_bpp_dither. */
7864                         BUG();
7865                 }
7866         }
7867
7868         if (HAS_PIPE_CXSR(dev)) {
7869                 if (intel_crtc->lowfreq_avail) {
7870                         DRM_DEBUG_KMS("enabling CxSR downclocking\n");
7871                         pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
7872                 } else {
7873                         DRM_DEBUG_KMS("disabling CxSR downclocking\n");
7874                 }
7875         }
7876
7877         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
7878                 if (INTEL_INFO(dev)->gen < 4 ||
7879                     intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
7880                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
7881                 else
7882                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
7883         } else
7884                 pipeconf |= PIPECONF_PROGRESSIVE;
7885
7886         if (IS_VALLEYVIEW(dev) && intel_crtc->config->limited_color_range)
7887                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
7888
7889         I915_WRITE(PIPECONF(intel_crtc->pipe), pipeconf);
7890         POSTING_READ(PIPECONF(intel_crtc->pipe));
7891 }
7892
7893 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
7894                                    struct intel_crtc_state *crtc_state)
7895 {
7896         struct drm_device *dev = crtc->base.dev;
7897         struct drm_i915_private *dev_priv = dev->dev_private;
7898         int refclk, num_connectors = 0;
7899         intel_clock_t clock;
7900         bool ok;
7901         bool is_dsi = false;
7902         struct intel_encoder *encoder;
7903         const intel_limit_t *limit;
7904         struct drm_atomic_state *state = crtc_state->base.state;
7905         struct drm_connector *connector;
7906         struct drm_connector_state *connector_state;
7907         int i;
7908
7909         memset(&crtc_state->dpll_hw_state, 0,
7910                sizeof(crtc_state->dpll_hw_state));
7911
7912         for_each_connector_in_state(state, connector, connector_state, i) {
7913                 if (connector_state->crtc != &crtc->base)
7914                         continue;
7915
7916                 encoder = to_intel_encoder(connector_state->best_encoder);
7917
7918                 switch (encoder->type) {
7919                 case INTEL_OUTPUT_DSI:
7920                         is_dsi = true;
7921                         break;
7922                 default:
7923                         break;
7924                 }
7925
7926                 num_connectors++;
7927         }
7928
7929         if (is_dsi)
7930                 return 0;
7931
7932         if (!crtc_state->clock_set) {
7933                 refclk = i9xx_get_refclk(crtc_state, num_connectors);
7934
7935                 /*
7936                  * Returns a set of divisors for the desired target clock with
7937                  * the given refclk, or FALSE.  The returned values represent
7938                  * the clock equation: reflck * (5 * (m1 + 2) + (m2 + 2)) / (n +
7939                  * 2) / p1 / p2.
7940                  */
7941                 limit = intel_limit(crtc_state, refclk);
7942                 ok = dev_priv->display.find_dpll(limit, crtc_state,
7943                                                  crtc_state->port_clock,
7944                                                  refclk, NULL, &clock);
7945                 if (!ok) {
7946                         DRM_ERROR("Couldn't find PLL settings for mode!\n");
7947                         return -EINVAL;
7948                 }
7949
7950                 /* Compat-code for transition, will disappear. */
7951                 crtc_state->dpll.n = clock.n;
7952                 crtc_state->dpll.m1 = clock.m1;
7953                 crtc_state->dpll.m2 = clock.m2;
7954                 crtc_state->dpll.p1 = clock.p1;
7955                 crtc_state->dpll.p2 = clock.p2;
7956         }
7957
7958         if (IS_GEN2(dev)) {
7959                 i8xx_compute_dpll(crtc, crtc_state, NULL,
7960                                   num_connectors);
7961         } else if (IS_CHERRYVIEW(dev)) {
7962                 chv_compute_dpll(crtc, crtc_state);
7963         } else if (IS_VALLEYVIEW(dev)) {
7964                 vlv_compute_dpll(crtc, crtc_state);
7965         } else {
7966                 i9xx_compute_dpll(crtc, crtc_state, NULL,
7967                                   num_connectors);
7968         }
7969
7970         return 0;
7971 }
7972
7973 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
7974                                  struct intel_crtc_state *pipe_config)
7975 {
7976         struct drm_device *dev = crtc->base.dev;
7977         struct drm_i915_private *dev_priv = dev->dev_private;
7978         uint32_t tmp;
7979
7980         if (INTEL_INFO(dev)->gen <= 3 && (IS_I830(dev) || !IS_MOBILE(dev)))
7981                 return;
7982
7983         tmp = I915_READ(PFIT_CONTROL);
7984         if (!(tmp & PFIT_ENABLE))
7985                 return;
7986
7987         /* Check whether the pfit is attached to our pipe. */
7988         if (INTEL_INFO(dev)->gen < 4) {
7989                 if (crtc->pipe != PIPE_B)
7990                         return;
7991         } else {
7992                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
7993                         return;
7994         }
7995
7996         pipe_config->gmch_pfit.control = tmp;
7997         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
7998         if (INTEL_INFO(dev)->gen < 5)
7999                 pipe_config->gmch_pfit.lvds_border_bits =
8000                         I915_READ(LVDS) & LVDS_BORDER_ENABLE;
8001 }
8002
8003 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
8004                                struct intel_crtc_state *pipe_config)
8005 {
8006         struct drm_device *dev = crtc->base.dev;
8007         struct drm_i915_private *dev_priv = dev->dev_private;
8008         int pipe = pipe_config->cpu_transcoder;
8009         intel_clock_t clock;
8010         u32 mdiv;
8011         int refclk = 100000;
8012
8013         /* In case of MIPI DPLL will not even be used */
8014         if (!(pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE))
8015                 return;
8016
8017         mutex_lock(&dev_priv->sb_lock);
8018         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
8019         mutex_unlock(&dev_priv->sb_lock);
8020
8021         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
8022         clock.m2 = mdiv & DPIO_M2DIV_MASK;
8023         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
8024         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
8025         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
8026
8027         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
8028 }
8029
8030 static void
8031 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8032                               struct intel_initial_plane_config *plane_config)
8033 {
8034         struct drm_device *dev = crtc->base.dev;
8035         struct drm_i915_private *dev_priv = dev->dev_private;
8036         u32 val, base, offset;
8037         int pipe = crtc->pipe, plane = crtc->plane;
8038         int fourcc, pixel_format;
8039         unsigned int aligned_height;
8040         struct drm_framebuffer *fb;
8041         struct intel_framebuffer *intel_fb;
8042
8043         val = I915_READ(DSPCNTR(plane));
8044         if (!(val & DISPLAY_PLANE_ENABLE))
8045                 return;
8046
8047         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8048         if (!intel_fb) {
8049                 DRM_DEBUG_KMS("failed to alloc fb\n");
8050                 return;
8051         }
8052
8053         fb = &intel_fb->base;
8054
8055         if (INTEL_INFO(dev)->gen >= 4) {
8056                 if (val & DISPPLANE_TILED) {
8057                         plane_config->tiling = I915_TILING_X;
8058                         fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
8059                 }
8060         }
8061
8062         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
8063         fourcc = i9xx_format_to_fourcc(pixel_format);
8064         fb->pixel_format = fourcc;
8065         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
8066
8067         if (INTEL_INFO(dev)->gen >= 4) {
8068                 if (plane_config->tiling)
8069                         offset = I915_READ(DSPTILEOFF(plane));
8070                 else
8071                         offset = I915_READ(DSPLINOFF(plane));
8072                 base = I915_READ(DSPSURF(plane)) & 0xfffff000;
8073         } else {
8074                 base = I915_READ(DSPADDR(plane));
8075         }
8076         plane_config->base = base;
8077
8078         val = I915_READ(PIPESRC(pipe));
8079         fb->width = ((val >> 16) & 0xfff) + 1;
8080         fb->height = ((val >> 0) & 0xfff) + 1;
8081
8082         val = I915_READ(DSPSTRIDE(pipe));
8083         fb->pitches[0] = val & 0xffffffc0;
8084
8085         aligned_height = intel_fb_align_height(dev, fb->height,
8086                                                fb->pixel_format,
8087                                                fb->modifier[0]);
8088
8089         plane_config->size = fb->pitches[0] * aligned_height;
8090
8091         DRM_DEBUG_KMS("pipe/plane %c/%d with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8092                       pipe_name(pipe), plane, fb->width, fb->height,
8093                       fb->bits_per_pixel, base, fb->pitches[0],
8094                       plane_config->size);
8095
8096         plane_config->fb = intel_fb;
8097 }
8098
8099 static void chv_crtc_clock_get(struct intel_crtc *crtc,
8100                                struct intel_crtc_state *pipe_config)
8101 {
8102         struct drm_device *dev = crtc->base.dev;
8103         struct drm_i915_private *dev_priv = dev->dev_private;
8104         int pipe = pipe_config->cpu_transcoder;
8105         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8106         intel_clock_t clock;
8107         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
8108         int refclk = 100000;
8109
8110         mutex_lock(&dev_priv->sb_lock);
8111         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
8112         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
8113         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
8114         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
8115         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8116         mutex_unlock(&dev_priv->sb_lock);
8117
8118         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
8119         clock.m2 = (pll_dw0 & 0xff) << 22;
8120         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
8121                 clock.m2 |= pll_dw2 & 0x3fffff;
8122         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
8123         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
8124         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
8125
8126         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
8127 }
8128
8129 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
8130                                  struct intel_crtc_state *pipe_config)
8131 {
8132         struct drm_device *dev = crtc->base.dev;
8133         struct drm_i915_private *dev_priv = dev->dev_private;
8134         uint32_t tmp;
8135
8136         if (!intel_display_power_is_enabled(dev_priv,
8137                                             POWER_DOMAIN_PIPE(crtc->pipe)))
8138                 return false;
8139
8140         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
8141         pipe_config->shared_dpll = DPLL_ID_PRIVATE;
8142
8143         tmp = I915_READ(PIPECONF(crtc->pipe));
8144         if (!(tmp & PIPECONF_ENABLE))
8145                 return false;
8146
8147         if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
8148                 switch (tmp & PIPECONF_BPC_MASK) {
8149                 case PIPECONF_6BPC:
8150                         pipe_config->pipe_bpp = 18;
8151                         break;
8152                 case PIPECONF_8BPC:
8153                         pipe_config->pipe_bpp = 24;
8154                         break;
8155                 case PIPECONF_10BPC:
8156                         pipe_config->pipe_bpp = 30;
8157                         break;
8158                 default:
8159                         break;
8160                 }
8161         }
8162
8163         if (IS_VALLEYVIEW(dev) && (tmp & PIPECONF_COLOR_RANGE_SELECT))
8164                 pipe_config->limited_color_range = true;
8165
8166         if (INTEL_INFO(dev)->gen < 4)
8167                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
8168
8169         intel_get_pipe_timings(crtc, pipe_config);
8170
8171         i9xx_get_pfit_config(crtc, pipe_config);
8172
8173         if (INTEL_INFO(dev)->gen >= 4) {
8174                 tmp = I915_READ(DPLL_MD(crtc->pipe));
8175                 pipe_config->pixel_multiplier =
8176                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
8177                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
8178                 pipe_config->dpll_hw_state.dpll_md = tmp;
8179         } else if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
8180                 tmp = I915_READ(DPLL(crtc->pipe));
8181                 pipe_config->pixel_multiplier =
8182                         ((tmp & SDVO_MULTIPLIER_MASK)
8183                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
8184         } else {
8185                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
8186                  * port and will be fixed up in the encoder->get_config
8187                  * function. */
8188                 pipe_config->pixel_multiplier = 1;
8189         }
8190         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
8191         if (!IS_VALLEYVIEW(dev)) {
8192                 /*
8193                  * DPLL_DVO_2X_MODE must be enabled for both DPLLs
8194                  * on 830. Filter it out here so that we don't
8195                  * report errors due to that.
8196                  */
8197                 if (IS_I830(dev))
8198                         pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
8199
8200                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
8201                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
8202         } else {
8203                 /* Mask out read-only status bits. */
8204                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
8205                                                      DPLL_PORTC_READY_MASK |
8206                                                      DPLL_PORTB_READY_MASK);
8207         }
8208
8209         if (IS_CHERRYVIEW(dev))
8210                 chv_crtc_clock_get(crtc, pipe_config);
8211         else if (IS_VALLEYVIEW(dev))
8212                 vlv_crtc_clock_get(crtc, pipe_config);
8213         else
8214                 i9xx_crtc_clock_get(crtc, pipe_config);
8215
8216         /*
8217          * Normally the dotclock is filled in by the encoder .get_config()
8218          * but in case the pipe is enabled w/o any ports we need a sane
8219          * default.
8220          */
8221         pipe_config->base.adjusted_mode.crtc_clock =
8222                 pipe_config->port_clock / pipe_config->pixel_multiplier;
8223
8224         return true;
8225 }
8226
8227 static void ironlake_init_pch_refclk(struct drm_device *dev)
8228 {
8229         struct drm_i915_private *dev_priv = dev->dev_private;
8230         struct intel_encoder *encoder;
8231         int i;
8232         u32 val, final;
8233         bool has_lvds = false;
8234         bool has_cpu_edp = false;
8235         bool has_panel = false;
8236         bool has_ck505 = false;
8237         bool can_ssc = false;
8238         bool using_ssc_source = false;
8239
8240         /* We need to take the global config into account */
8241         for_each_intel_encoder(dev, encoder) {
8242                 switch (encoder->type) {
8243                 case INTEL_OUTPUT_LVDS:
8244                         has_panel = true;
8245                         has_lvds = true;
8246                         break;
8247                 case INTEL_OUTPUT_EDP:
8248                         has_panel = true;
8249                         if (enc_to_dig_port(&encoder->base)->port == PORT_A)
8250                                 has_cpu_edp = true;
8251                         break;
8252                 default:
8253                         break;
8254                 }
8255         }
8256
8257         if (HAS_PCH_IBX(dev)) {
8258                 has_ck505 = dev_priv->vbt.display_clock_mode;
8259                 can_ssc = has_ck505;
8260         } else {
8261                 has_ck505 = false;
8262                 can_ssc = true;
8263         }
8264
8265         /* Check if any DPLLs are using the SSC source */
8266         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
8267                 u32 temp = I915_READ(PCH_DPLL(i));
8268
8269                 if (!(temp & DPLL_VCO_ENABLE))
8270                         continue;
8271
8272                 if ((temp & PLL_REF_INPUT_MASK) ==
8273                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
8274                         using_ssc_source = true;
8275                         break;
8276                 }
8277         }
8278
8279         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
8280                       has_panel, has_lvds, has_ck505, using_ssc_source);
8281
8282         /* Ironlake: try to setup display ref clock before DPLL
8283          * enabling. This is only under driver's control after
8284          * PCH B stepping, previous chipset stepping should be
8285          * ignoring this setting.
8286          */
8287         val = I915_READ(PCH_DREF_CONTROL);
8288
8289         /* As we must carefully and slowly disable/enable each source in turn,
8290          * compute the final state we want first and check if we need to
8291          * make any changes at all.
8292          */
8293         final = val;
8294         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8295         if (has_ck505)
8296                 final |= DREF_NONSPREAD_CK505_ENABLE;
8297         else
8298                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8299
8300         final &= ~DREF_SSC_SOURCE_MASK;
8301         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8302         final &= ~DREF_SSC1_ENABLE;
8303
8304         if (has_panel) {
8305                 final |= DREF_SSC_SOURCE_ENABLE;
8306
8307                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8308                         final |= DREF_SSC1_ENABLE;
8309
8310                 if (has_cpu_edp) {
8311                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8312                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8313                         else
8314                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8315                 } else
8316                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8317         } else if (using_ssc_source) {
8318                 final |= DREF_SSC_SOURCE_ENABLE;
8319                 final |= DREF_SSC1_ENABLE;
8320         }
8321
8322         if (final == val)
8323                 return;
8324
8325         /* Always enable nonspread source */
8326         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8327
8328         if (has_ck505)
8329                 val |= DREF_NONSPREAD_CK505_ENABLE;
8330         else
8331                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8332
8333         if (has_panel) {
8334                 val &= ~DREF_SSC_SOURCE_MASK;
8335                 val |= DREF_SSC_SOURCE_ENABLE;
8336
8337                 /* SSC must be turned on before enabling the CPU output  */
8338                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8339                         DRM_DEBUG_KMS("Using SSC on panel\n");
8340                         val |= DREF_SSC1_ENABLE;
8341                 } else
8342                         val &= ~DREF_SSC1_ENABLE;
8343
8344                 /* Get SSC going before enabling the outputs */
8345                 I915_WRITE(PCH_DREF_CONTROL, val);
8346                 POSTING_READ(PCH_DREF_CONTROL);
8347                 udelay(200);
8348
8349                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8350
8351                 /* Enable CPU source on CPU attached eDP */
8352                 if (has_cpu_edp) {
8353                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8354                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
8355                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8356                         } else
8357                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8358                 } else
8359                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8360
8361                 I915_WRITE(PCH_DREF_CONTROL, val);
8362                 POSTING_READ(PCH_DREF_CONTROL);
8363                 udelay(200);
8364         } else {
8365                 DRM_DEBUG_KMS("Disabling CPU source output\n");
8366
8367                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8368
8369                 /* Turn off CPU output */
8370                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8371
8372                 I915_WRITE(PCH_DREF_CONTROL, val);
8373                 POSTING_READ(PCH_DREF_CONTROL);
8374                 udelay(200);
8375
8376                 if (!using_ssc_source) {
8377                         DRM_DEBUG_KMS("Disabling SSC source\n");
8378
8379                         /* Turn off the SSC source */
8380                         val &= ~DREF_SSC_SOURCE_MASK;
8381                         val |= DREF_SSC_SOURCE_DISABLE;
8382
8383                         /* Turn off SSC1 */
8384                         val &= ~DREF_SSC1_ENABLE;
8385
8386                         I915_WRITE(PCH_DREF_CONTROL, val);
8387                         POSTING_READ(PCH_DREF_CONTROL);
8388                         udelay(200);
8389                 }
8390         }
8391
8392         BUG_ON(val != final);
8393 }
8394
8395 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
8396 {
8397         uint32_t tmp;
8398
8399         tmp = I915_READ(SOUTH_CHICKEN2);
8400         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
8401         I915_WRITE(SOUTH_CHICKEN2, tmp);
8402
8403         if (wait_for_atomic_us(I915_READ(SOUTH_CHICKEN2) &
8404                                FDI_MPHY_IOSFSB_RESET_STATUS, 100))
8405                 DRM_ERROR("FDI mPHY reset assert timeout\n");
8406
8407         tmp = I915_READ(SOUTH_CHICKEN2);
8408         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
8409         I915_WRITE(SOUTH_CHICKEN2, tmp);
8410
8411         if (wait_for_atomic_us((I915_READ(SOUTH_CHICKEN2) &
8412                                 FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
8413                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
8414 }
8415
8416 /* WaMPhyProgramming:hsw */
8417 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
8418 {
8419         uint32_t tmp;
8420
8421         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
8422         tmp &= ~(0xFF << 24);
8423         tmp |= (0x12 << 24);
8424         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
8425
8426         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
8427         tmp |= (1 << 11);
8428         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
8429
8430         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
8431         tmp |= (1 << 11);
8432         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
8433
8434         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
8435         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8436         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
8437
8438         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
8439         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8440         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
8441
8442         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
8443         tmp &= ~(7 << 13);
8444         tmp |= (5 << 13);
8445         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
8446
8447         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
8448         tmp &= ~(7 << 13);
8449         tmp |= (5 << 13);
8450         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
8451
8452         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
8453         tmp &= ~0xFF;
8454         tmp |= 0x1C;
8455         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
8456
8457         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
8458         tmp &= ~0xFF;
8459         tmp |= 0x1C;
8460         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
8461
8462         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
8463         tmp &= ~(0xFF << 16);
8464         tmp |= (0x1C << 16);
8465         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
8466
8467         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
8468         tmp &= ~(0xFF << 16);
8469         tmp |= (0x1C << 16);
8470         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
8471
8472         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
8473         tmp |= (1 << 27);
8474         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
8475
8476         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
8477         tmp |= (1 << 27);
8478         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
8479
8480         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
8481         tmp &= ~(0xF << 28);
8482         tmp |= (4 << 28);
8483         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
8484
8485         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
8486         tmp &= ~(0xF << 28);
8487         tmp |= (4 << 28);
8488         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
8489 }
8490
8491 /* Implements 3 different sequences from BSpec chapter "Display iCLK
8492  * Programming" based on the parameters passed:
8493  * - Sequence to enable CLKOUT_DP
8494  * - Sequence to enable CLKOUT_DP without spread
8495  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
8496  */
8497 static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread,
8498                                  bool with_fdi)
8499 {
8500         struct drm_i915_private *dev_priv = dev->dev_private;
8501         uint32_t reg, tmp;
8502
8503         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
8504                 with_spread = true;
8505         if (WARN(HAS_PCH_LPT_LP(dev) && with_fdi, "LP PCH doesn't have FDI\n"))
8506                 with_fdi = false;
8507
8508         mutex_lock(&dev_priv->sb_lock);
8509
8510         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8511         tmp &= ~SBI_SSCCTL_DISABLE;
8512         tmp |= SBI_SSCCTL_PATHALT;
8513         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8514
8515         udelay(24);
8516
8517         if (with_spread) {
8518                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8519                 tmp &= ~SBI_SSCCTL_PATHALT;
8520                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8521
8522                 if (with_fdi) {
8523                         lpt_reset_fdi_mphy(dev_priv);
8524                         lpt_program_fdi_mphy(dev_priv);
8525                 }
8526         }
8527
8528         reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
8529         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8530         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8531         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8532
8533         mutex_unlock(&dev_priv->sb_lock);
8534 }
8535
8536 /* Sequence to disable CLKOUT_DP */
8537 static void lpt_disable_clkout_dp(struct drm_device *dev)
8538 {
8539         struct drm_i915_private *dev_priv = dev->dev_private;
8540         uint32_t reg, tmp;
8541
8542         mutex_lock(&dev_priv->sb_lock);
8543
8544         reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
8545         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8546         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8547         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8548
8549         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8550         if (!(tmp & SBI_SSCCTL_DISABLE)) {
8551                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
8552                         tmp |= SBI_SSCCTL_PATHALT;
8553                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8554                         udelay(32);
8555                 }
8556                 tmp |= SBI_SSCCTL_DISABLE;
8557                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8558         }
8559
8560         mutex_unlock(&dev_priv->sb_lock);
8561 }
8562
8563 static void lpt_init_pch_refclk(struct drm_device *dev)
8564 {
8565         struct intel_encoder *encoder;
8566         bool has_vga = false;
8567
8568         for_each_intel_encoder(dev, encoder) {
8569                 switch (encoder->type) {
8570                 case INTEL_OUTPUT_ANALOG:
8571                         has_vga = true;
8572                         break;
8573                 default:
8574                         break;
8575                 }
8576         }
8577
8578         if (has_vga)
8579                 lpt_enable_clkout_dp(dev, true, true);
8580         else
8581                 lpt_disable_clkout_dp(dev);
8582 }
8583
8584 /*
8585  * Initialize reference clocks when the driver loads
8586  */
8587 void intel_init_pch_refclk(struct drm_device *dev)
8588 {
8589         if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
8590                 ironlake_init_pch_refclk(dev);
8591         else if (HAS_PCH_LPT(dev))
8592                 lpt_init_pch_refclk(dev);
8593 }
8594
8595 static int ironlake_get_refclk(struct intel_crtc_state *crtc_state)
8596 {
8597         struct drm_device *dev = crtc_state->base.crtc->dev;
8598         struct drm_i915_private *dev_priv = dev->dev_private;
8599         struct drm_atomic_state *state = crtc_state->base.state;
8600         struct drm_connector *connector;
8601         struct drm_connector_state *connector_state;
8602         struct intel_encoder *encoder;
8603         int num_connectors = 0, i;
8604         bool is_lvds = false;
8605
8606         for_each_connector_in_state(state, connector, connector_state, i) {
8607                 if (connector_state->crtc != crtc_state->base.crtc)
8608                         continue;
8609
8610                 encoder = to_intel_encoder(connector_state->best_encoder);
8611
8612                 switch (encoder->type) {
8613                 case INTEL_OUTPUT_LVDS:
8614                         is_lvds = true;
8615                         break;
8616                 default:
8617                         break;
8618                 }
8619                 num_connectors++;
8620         }
8621
8622         if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
8623                 DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
8624                               dev_priv->vbt.lvds_ssc_freq);
8625                 return dev_priv->vbt.lvds_ssc_freq;
8626         }
8627
8628         return 120000;
8629 }
8630
8631 static void ironlake_set_pipeconf(struct drm_crtc *crtc)
8632 {
8633         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
8634         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8635         int pipe = intel_crtc->pipe;
8636         uint32_t val;
8637
8638         val = 0;
8639
8640         switch (intel_crtc->config->pipe_bpp) {
8641         case 18:
8642                 val |= PIPECONF_6BPC;
8643                 break;
8644         case 24:
8645                 val |= PIPECONF_8BPC;
8646                 break;
8647         case 30:
8648                 val |= PIPECONF_10BPC;
8649                 break;
8650         case 36:
8651                 val |= PIPECONF_12BPC;
8652                 break;
8653         default:
8654                 /* Case prevented by intel_choose_pipe_bpp_dither. */
8655                 BUG();
8656         }
8657
8658         if (intel_crtc->config->dither)
8659                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8660
8661         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8662                 val |= PIPECONF_INTERLACED_ILK;
8663         else
8664                 val |= PIPECONF_PROGRESSIVE;
8665
8666         if (intel_crtc->config->limited_color_range)
8667                 val |= PIPECONF_COLOR_RANGE_SELECT;
8668
8669         I915_WRITE(PIPECONF(pipe), val);
8670         POSTING_READ(PIPECONF(pipe));
8671 }
8672
8673 /*
8674  * Set up the pipe CSC unit.
8675  *
8676  * Currently only full range RGB to limited range RGB conversion
8677  * is supported, but eventually this should handle various
8678  * RGB<->YCbCr scenarios as well.
8679  */
8680 static void intel_set_pipe_csc(struct drm_crtc *crtc)
8681 {
8682         struct drm_device *dev = crtc->dev;
8683         struct drm_i915_private *dev_priv = dev->dev_private;
8684         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8685         int pipe = intel_crtc->pipe;
8686         uint16_t coeff = 0x7800; /* 1.0 */
8687
8688         /*
8689          * TODO: Check what kind of values actually come out of the pipe
8690          * with these coeff/postoff values and adjust to get the best
8691          * accuracy. Perhaps we even need to take the bpc value into
8692          * consideration.
8693          */
8694
8695         if (intel_crtc->config->limited_color_range)
8696                 coeff = ((235 - 16) * (1 << 12) / 255) & 0xff8; /* 0.xxx... */
8697
8698         /*
8699          * GY/GU and RY/RU should be the other way around according
8700          * to BSpec, but reality doesn't agree. Just set them up in
8701          * a way that results in the correct picture.
8702          */
8703         I915_WRITE(PIPE_CSC_COEFF_RY_GY(pipe), coeff << 16);
8704         I915_WRITE(PIPE_CSC_COEFF_BY(pipe), 0);
8705
8706         I915_WRITE(PIPE_CSC_COEFF_RU_GU(pipe), coeff);
8707         I915_WRITE(PIPE_CSC_COEFF_BU(pipe), 0);
8708
8709         I915_WRITE(PIPE_CSC_COEFF_RV_GV(pipe), 0);
8710         I915_WRITE(PIPE_CSC_COEFF_BV(pipe), coeff << 16);
8711
8712         I915_WRITE(PIPE_CSC_PREOFF_HI(pipe), 0);
8713         I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), 0);
8714         I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), 0);
8715
8716         if (INTEL_INFO(dev)->gen > 6) {
8717                 uint16_t postoff = 0;
8718
8719                 if (intel_crtc->config->limited_color_range)
8720                         postoff = (16 * (1 << 12) / 255) & 0x1fff;
8721
8722                 I915_WRITE(PIPE_CSC_POSTOFF_HI(pipe), postoff);
8723                 I915_WRITE(PIPE_CSC_POSTOFF_ME(pipe), postoff);
8724                 I915_WRITE(PIPE_CSC_POSTOFF_LO(pipe), postoff);
8725
8726                 I915_WRITE(PIPE_CSC_MODE(pipe), 0);
8727         } else {
8728                 uint32_t mode = CSC_MODE_YUV_TO_RGB;
8729
8730                 if (intel_crtc->config->limited_color_range)
8731                         mode |= CSC_BLACK_SCREEN_OFFSET;
8732
8733                 I915_WRITE(PIPE_CSC_MODE(pipe), mode);
8734         }
8735 }
8736
8737 static void haswell_set_pipeconf(struct drm_crtc *crtc)
8738 {
8739         struct drm_device *dev = crtc->dev;
8740         struct drm_i915_private *dev_priv = dev->dev_private;
8741         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8742         enum pipe pipe = intel_crtc->pipe;
8743         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
8744         uint32_t val;
8745
8746         val = 0;
8747
8748         if (IS_HASWELL(dev) && intel_crtc->config->dither)
8749                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8750
8751         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8752                 val |= PIPECONF_INTERLACED_ILK;
8753         else
8754                 val |= PIPECONF_PROGRESSIVE;
8755
8756         I915_WRITE(PIPECONF(cpu_transcoder), val);
8757         POSTING_READ(PIPECONF(cpu_transcoder));
8758
8759         I915_WRITE(GAMMA_MODE(intel_crtc->pipe), GAMMA_MODE_MODE_8BIT);
8760         POSTING_READ(GAMMA_MODE(intel_crtc->pipe));
8761
8762         if (IS_BROADWELL(dev) || INTEL_INFO(dev)->gen >= 9) {
8763                 val = 0;
8764
8765                 switch (intel_crtc->config->pipe_bpp) {
8766                 case 18:
8767                         val |= PIPEMISC_DITHER_6_BPC;
8768                         break;
8769                 case 24:
8770                         val |= PIPEMISC_DITHER_8_BPC;
8771                         break;
8772                 case 30:
8773                         val |= PIPEMISC_DITHER_10_BPC;
8774                         break;
8775                 case 36:
8776                         val |= PIPEMISC_DITHER_12_BPC;
8777                         break;
8778                 default:
8779                         /* Case prevented by pipe_config_set_bpp. */
8780                         BUG();
8781                 }
8782
8783                 if (intel_crtc->config->dither)
8784                         val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
8785
8786                 I915_WRITE(PIPEMISC(pipe), val);
8787         }
8788 }
8789
8790 static bool ironlake_compute_clocks(struct drm_crtc *crtc,
8791                                     struct intel_crtc_state *crtc_state,
8792                                     intel_clock_t *clock,
8793                                     bool *has_reduced_clock,
8794                                     intel_clock_t *reduced_clock)
8795 {
8796         struct drm_device *dev = crtc->dev;
8797         struct drm_i915_private *dev_priv = dev->dev_private;
8798         int refclk;
8799         const intel_limit_t *limit;
8800         bool ret;
8801
8802         refclk = ironlake_get_refclk(crtc_state);
8803
8804         /*
8805          * Returns a set of divisors for the desired target clock with the given
8806          * refclk, or FALSE.  The returned values represent the clock equation:
8807          * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
8808          */
8809         limit = intel_limit(crtc_state, refclk);
8810         ret = dev_priv->display.find_dpll(limit, crtc_state,
8811                                           crtc_state->port_clock,
8812                                           refclk, NULL, clock);
8813         if (!ret)
8814                 return false;
8815
8816         return true;
8817 }
8818
8819 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
8820 {
8821         /*
8822          * Account for spread spectrum to avoid
8823          * oversubscribing the link. Max center spread
8824          * is 2.5%; use 5% for safety's sake.
8825          */
8826         u32 bps = target_clock * bpp * 21 / 20;
8827         return DIV_ROUND_UP(bps, link_bw * 8);
8828 }
8829
8830 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
8831 {
8832         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
8833 }
8834
8835 static uint32_t ironlake_compute_dpll(struct intel_crtc *intel_crtc,
8836                                       struct intel_crtc_state *crtc_state,
8837                                       u32 *fp,
8838                                       intel_clock_t *reduced_clock, u32 *fp2)
8839 {
8840         struct drm_crtc *crtc = &intel_crtc->base;
8841         struct drm_device *dev = crtc->dev;
8842         struct drm_i915_private *dev_priv = dev->dev_private;
8843         struct drm_atomic_state *state = crtc_state->base.state;
8844         struct drm_connector *connector;
8845         struct drm_connector_state *connector_state;
8846         struct intel_encoder *encoder;
8847         uint32_t dpll;
8848         int factor, num_connectors = 0, i;
8849         bool is_lvds = false, is_sdvo = false;
8850
8851         for_each_connector_in_state(state, connector, connector_state, i) {
8852                 if (connector_state->crtc != crtc_state->base.crtc)
8853                         continue;
8854
8855                 encoder = to_intel_encoder(connector_state->best_encoder);
8856
8857                 switch (encoder->type) {
8858                 case INTEL_OUTPUT_LVDS:
8859                         is_lvds = true;
8860                         break;
8861                 case INTEL_OUTPUT_SDVO:
8862                 case INTEL_OUTPUT_HDMI:
8863                         is_sdvo = true;
8864                         break;
8865                 default:
8866                         break;
8867                 }
8868
8869                 num_connectors++;
8870         }
8871
8872         /* Enable autotuning of the PLL clock (if permissible) */
8873         factor = 21;
8874         if (is_lvds) {
8875                 if ((intel_panel_use_ssc(dev_priv) &&
8876                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
8877                     (HAS_PCH_IBX(dev) && intel_is_dual_link_lvds(dev)))
8878                         factor = 25;
8879         } else if (crtc_state->sdvo_tv_clock)
8880                 factor = 20;
8881
8882         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
8883                 *fp |= FP_CB_TUNE;
8884
8885         if (fp2 && (reduced_clock->m < factor * reduced_clock->n))
8886                 *fp2 |= FP_CB_TUNE;
8887
8888         dpll = 0;
8889
8890         if (is_lvds)
8891                 dpll |= DPLLB_MODE_LVDS;
8892         else
8893                 dpll |= DPLLB_MODE_DAC_SERIAL;
8894
8895         dpll |= (crtc_state->pixel_multiplier - 1)
8896                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
8897
8898         if (is_sdvo)
8899                 dpll |= DPLL_SDVO_HIGH_SPEED;
8900         if (crtc_state->has_dp_encoder)
8901                 dpll |= DPLL_SDVO_HIGH_SPEED;
8902
8903         /* compute bitmask from p1 value */
8904         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8905         /* also FPA1 */
8906         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8907
8908         switch (crtc_state->dpll.p2) {
8909         case 5:
8910                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8911                 break;
8912         case 7:
8913                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8914                 break;
8915         case 10:
8916                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8917                 break;
8918         case 14:
8919                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8920                 break;
8921         }
8922
8923         if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
8924                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8925         else
8926                 dpll |= PLL_REF_INPUT_DREFCLK;
8927
8928         return dpll | DPLL_VCO_ENABLE;
8929 }
8930
8931 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
8932                                        struct intel_crtc_state *crtc_state)
8933 {
8934         struct drm_device *dev = crtc->base.dev;
8935         intel_clock_t clock, reduced_clock;
8936         u32 dpll = 0, fp = 0, fp2 = 0;
8937         bool ok, has_reduced_clock = false;
8938         bool is_lvds = false;
8939         struct intel_shared_dpll *pll;
8940
8941         memset(&crtc_state->dpll_hw_state, 0,
8942                sizeof(crtc_state->dpll_hw_state));
8943
8944         is_lvds = intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS);
8945
8946         WARN(!(HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)),
8947              "Unexpected PCH type %d\n", INTEL_PCH_TYPE(dev));
8948
8949         ok = ironlake_compute_clocks(&crtc->base, crtc_state, &clock,
8950                                      &has_reduced_clock, &reduced_clock);
8951         if (!ok && !crtc_state->clock_set) {
8952                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8953                 return -EINVAL;
8954         }
8955         /* Compat-code for transition, will disappear. */
8956         if (!crtc_state->clock_set) {
8957                 crtc_state->dpll.n = clock.n;
8958                 crtc_state->dpll.m1 = clock.m1;
8959                 crtc_state->dpll.m2 = clock.m2;
8960                 crtc_state->dpll.p1 = clock.p1;
8961                 crtc_state->dpll.p2 = clock.p2;
8962         }
8963
8964         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
8965         if (crtc_state->has_pch_encoder) {
8966                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
8967                 if (has_reduced_clock)
8968                         fp2 = i9xx_dpll_compute_fp(&reduced_clock);
8969
8970                 dpll = ironlake_compute_dpll(crtc, crtc_state,
8971                                              &fp, &reduced_clock,
8972                                              has_reduced_clock ? &fp2 : NULL);
8973
8974                 crtc_state->dpll_hw_state.dpll = dpll;
8975                 crtc_state->dpll_hw_state.fp0 = fp;
8976                 if (has_reduced_clock)
8977                         crtc_state->dpll_hw_state.fp1 = fp2;
8978                 else
8979                         crtc_state->dpll_hw_state.fp1 = fp;
8980
8981                 pll = intel_get_shared_dpll(crtc, crtc_state);
8982                 if (pll == NULL) {
8983                         DRM_DEBUG_DRIVER("failed to find PLL for pipe %c\n",
8984                                          pipe_name(crtc->pipe));
8985                         return -EINVAL;
8986                 }
8987         }
8988
8989         if (is_lvds && has_reduced_clock)
8990                 crtc->lowfreq_avail = true;
8991         else
8992                 crtc->lowfreq_avail = false;
8993
8994         return 0;
8995 }
8996
8997 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
8998                                          struct intel_link_m_n *m_n)
8999 {
9000         struct drm_device *dev = crtc->base.dev;
9001         struct drm_i915_private *dev_priv = dev->dev_private;
9002         enum pipe pipe = crtc->pipe;
9003
9004         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
9005         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
9006         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
9007                 & ~TU_SIZE_MASK;
9008         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
9009         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
9010                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9011 }
9012
9013 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
9014                                          enum transcoder transcoder,
9015                                          struct intel_link_m_n *m_n,
9016                                          struct intel_link_m_n *m2_n2)
9017 {
9018         struct drm_device *dev = crtc->base.dev;
9019         struct drm_i915_private *dev_priv = dev->dev_private;
9020         enum pipe pipe = crtc->pipe;
9021
9022         if (INTEL_INFO(dev)->gen >= 5) {
9023                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
9024                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
9025                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
9026                         & ~TU_SIZE_MASK;
9027                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
9028                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
9029                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9030                 /* Read M2_N2 registers only for gen < 8 (M2_N2 available for
9031                  * gen < 8) and if DRRS is supported (to make sure the
9032                  * registers are not unnecessarily read).
9033                  */
9034                 if (m2_n2 && INTEL_INFO(dev)->gen < 8 &&
9035                         crtc->config->has_drrs) {
9036                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
9037                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
9038                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
9039                                         & ~TU_SIZE_MASK;
9040                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
9041                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
9042                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9043                 }
9044         } else {
9045                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
9046                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
9047                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
9048                         & ~TU_SIZE_MASK;
9049                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
9050                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
9051                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9052         }
9053 }
9054
9055 void intel_dp_get_m_n(struct intel_crtc *crtc,
9056                       struct intel_crtc_state *pipe_config)
9057 {
9058         if (pipe_config->has_pch_encoder)
9059                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
9060         else
9061                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9062                                              &pipe_config->dp_m_n,
9063                                              &pipe_config->dp_m2_n2);
9064 }
9065
9066 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
9067                                         struct intel_crtc_state *pipe_config)
9068 {
9069         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9070                                      &pipe_config->fdi_m_n, NULL);
9071 }
9072
9073 static void skylake_get_pfit_config(struct intel_crtc *crtc,
9074                                     struct intel_crtc_state *pipe_config)
9075 {
9076         struct drm_device *dev = crtc->base.dev;
9077         struct drm_i915_private *dev_priv = dev->dev_private;
9078         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
9079         uint32_t ps_ctrl = 0;
9080         int id = -1;
9081         int i;
9082
9083         /* find scaler attached to this pipe */
9084         for (i = 0; i < crtc->num_scalers; i++) {
9085                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
9086                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
9087                         id = i;
9088                         pipe_config->pch_pfit.enabled = true;
9089                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
9090                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
9091                         break;
9092                 }
9093         }
9094
9095         scaler_state->scaler_id = id;
9096         if (id >= 0) {
9097                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
9098         } else {
9099                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
9100         }
9101 }
9102
9103 static void
9104 skylake_get_initial_plane_config(struct intel_crtc *crtc,
9105                                  struct intel_initial_plane_config *plane_config)
9106 {
9107         struct drm_device *dev = crtc->base.dev;
9108         struct drm_i915_private *dev_priv = dev->dev_private;
9109         u32 val, base, offset, stride_mult, tiling;
9110         int pipe = crtc->pipe;
9111         int fourcc, pixel_format;
9112         unsigned int aligned_height;
9113         struct drm_framebuffer *fb;
9114         struct intel_framebuffer *intel_fb;
9115
9116         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9117         if (!intel_fb) {
9118                 DRM_DEBUG_KMS("failed to alloc fb\n");
9119                 return;
9120         }
9121
9122         fb = &intel_fb->base;
9123
9124         val = I915_READ(PLANE_CTL(pipe, 0));
9125         if (!(val & PLANE_CTL_ENABLE))
9126                 goto error;
9127
9128         pixel_format = val & PLANE_CTL_FORMAT_MASK;
9129         fourcc = skl_format_to_fourcc(pixel_format,
9130                                       val & PLANE_CTL_ORDER_RGBX,
9131                                       val & PLANE_CTL_ALPHA_MASK);
9132         fb->pixel_format = fourcc;
9133         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
9134
9135         tiling = val & PLANE_CTL_TILED_MASK;
9136         switch (tiling) {
9137         case PLANE_CTL_TILED_LINEAR:
9138                 fb->modifier[0] = DRM_FORMAT_MOD_NONE;
9139                 break;
9140         case PLANE_CTL_TILED_X:
9141                 plane_config->tiling = I915_TILING_X;
9142                 fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
9143                 break;
9144         case PLANE_CTL_TILED_Y:
9145                 fb->modifier[0] = I915_FORMAT_MOD_Y_TILED;
9146                 break;
9147         case PLANE_CTL_TILED_YF:
9148                 fb->modifier[0] = I915_FORMAT_MOD_Yf_TILED;
9149                 break;
9150         default:
9151                 MISSING_CASE(tiling);
9152                 goto error;
9153         }
9154
9155         base = I915_READ(PLANE_SURF(pipe, 0)) & 0xfffff000;
9156         plane_config->base = base;
9157
9158         offset = I915_READ(PLANE_OFFSET(pipe, 0));
9159
9160         val = I915_READ(PLANE_SIZE(pipe, 0));
9161         fb->height = ((val >> 16) & 0xfff) + 1;
9162         fb->width = ((val >> 0) & 0x1fff) + 1;
9163
9164         val = I915_READ(PLANE_STRIDE(pipe, 0));
9165         stride_mult = intel_fb_stride_alignment(dev, fb->modifier[0],
9166                                                 fb->pixel_format);
9167         fb->pitches[0] = (val & 0x3ff) * stride_mult;
9168
9169         aligned_height = intel_fb_align_height(dev, fb->height,
9170                                                fb->pixel_format,
9171                                                fb->modifier[0]);
9172
9173         plane_config->size = fb->pitches[0] * aligned_height;
9174
9175         DRM_DEBUG_KMS("pipe %c with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9176                       pipe_name(pipe), fb->width, fb->height,
9177                       fb->bits_per_pixel, base, fb->pitches[0],
9178                       plane_config->size);
9179
9180         plane_config->fb = intel_fb;
9181         return;
9182
9183 error:
9184         kfree(fb);
9185 }
9186
9187 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
9188                                      struct intel_crtc_state *pipe_config)
9189 {
9190         struct drm_device *dev = crtc->base.dev;
9191         struct drm_i915_private *dev_priv = dev->dev_private;
9192         uint32_t tmp;
9193
9194         tmp = I915_READ(PF_CTL(crtc->pipe));
9195
9196         if (tmp & PF_ENABLE) {
9197                 pipe_config->pch_pfit.enabled = true;
9198                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
9199                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
9200
9201                 /* We currently do not free assignements of panel fitters on
9202                  * ivb/hsw (since we don't use the higher upscaling modes which
9203                  * differentiates them) so just WARN about this case for now. */
9204                 if (IS_GEN7(dev)) {
9205                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
9206                                 PF_PIPE_SEL_IVB(crtc->pipe));
9207                 }
9208         }
9209 }
9210
9211 static void
9212 ironlake_get_initial_plane_config(struct intel_crtc *crtc,
9213                                   struct intel_initial_plane_config *plane_config)
9214 {
9215         struct drm_device *dev = crtc->base.dev;
9216         struct drm_i915_private *dev_priv = dev->dev_private;
9217         u32 val, base, offset;
9218         int pipe = crtc->pipe;
9219         int fourcc, pixel_format;
9220         unsigned int aligned_height;
9221         struct drm_framebuffer *fb;
9222         struct intel_framebuffer *intel_fb;
9223
9224         val = I915_READ(DSPCNTR(pipe));
9225         if (!(val & DISPLAY_PLANE_ENABLE))
9226                 return;
9227
9228         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9229         if (!intel_fb) {
9230                 DRM_DEBUG_KMS("failed to alloc fb\n");
9231                 return;
9232         }
9233
9234         fb = &intel_fb->base;
9235
9236         if (INTEL_INFO(dev)->gen >= 4) {
9237                 if (val & DISPPLANE_TILED) {
9238                         plane_config->tiling = I915_TILING_X;
9239                         fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
9240                 }
9241         }
9242
9243         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
9244         fourcc = i9xx_format_to_fourcc(pixel_format);
9245         fb->pixel_format = fourcc;
9246         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
9247
9248         base = I915_READ(DSPSURF(pipe)) & 0xfffff000;
9249         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
9250                 offset = I915_READ(DSPOFFSET(pipe));
9251         } else {
9252                 if (plane_config->tiling)
9253                         offset = I915_READ(DSPTILEOFF(pipe));
9254                 else
9255                         offset = I915_READ(DSPLINOFF(pipe));
9256         }
9257         plane_config->base = base;
9258
9259         val = I915_READ(PIPESRC(pipe));
9260         fb->width = ((val >> 16) & 0xfff) + 1;
9261         fb->height = ((val >> 0) & 0xfff) + 1;
9262
9263         val = I915_READ(DSPSTRIDE(pipe));
9264         fb->pitches[0] = val & 0xffffffc0;
9265
9266         aligned_height = intel_fb_align_height(dev, fb->height,
9267                                                fb->pixel_format,
9268                                                fb->modifier[0]);
9269
9270         plane_config->size = fb->pitches[0] * aligned_height;
9271
9272         DRM_DEBUG_KMS("pipe %c with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9273                       pipe_name(pipe), fb->width, fb->height,
9274                       fb->bits_per_pixel, base, fb->pitches[0],
9275                       plane_config->size);
9276
9277         plane_config->fb = intel_fb;
9278 }
9279
9280 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
9281                                      struct intel_crtc_state *pipe_config)
9282 {
9283         struct drm_device *dev = crtc->base.dev;
9284         struct drm_i915_private *dev_priv = dev->dev_private;
9285         uint32_t tmp;
9286
9287         if (!intel_display_power_is_enabled(dev_priv,
9288                                             POWER_DOMAIN_PIPE(crtc->pipe)))
9289                 return false;
9290
9291         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9292         pipe_config->shared_dpll = DPLL_ID_PRIVATE;
9293
9294         tmp = I915_READ(PIPECONF(crtc->pipe));
9295         if (!(tmp & PIPECONF_ENABLE))
9296                 return false;
9297
9298         switch (tmp & PIPECONF_BPC_MASK) {
9299         case PIPECONF_6BPC:
9300                 pipe_config->pipe_bpp = 18;
9301                 break;
9302         case PIPECONF_8BPC:
9303                 pipe_config->pipe_bpp = 24;
9304                 break;
9305         case PIPECONF_10BPC:
9306                 pipe_config->pipe_bpp = 30;
9307                 break;
9308         case PIPECONF_12BPC:
9309                 pipe_config->pipe_bpp = 36;
9310                 break;
9311         default:
9312                 break;
9313         }
9314
9315         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
9316                 pipe_config->limited_color_range = true;
9317
9318         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
9319                 struct intel_shared_dpll *pll;
9320
9321                 pipe_config->has_pch_encoder = true;
9322
9323                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
9324                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9325                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9326
9327                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9328
9329                 if (HAS_PCH_IBX(dev_priv->dev)) {
9330                         pipe_config->shared_dpll =
9331                                 (enum intel_dpll_id) crtc->pipe;
9332                 } else {
9333                         tmp = I915_READ(PCH_DPLL_SEL);
9334                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
9335                                 pipe_config->shared_dpll = DPLL_ID_PCH_PLL_B;
9336                         else
9337                                 pipe_config->shared_dpll = DPLL_ID_PCH_PLL_A;
9338                 }
9339
9340                 pll = &dev_priv->shared_dplls[pipe_config->shared_dpll];
9341
9342                 WARN_ON(!pll->get_hw_state(dev_priv, pll,
9343                                            &pipe_config->dpll_hw_state));
9344
9345                 tmp = pipe_config->dpll_hw_state.dpll;
9346                 pipe_config->pixel_multiplier =
9347                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
9348                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
9349
9350                 ironlake_pch_clock_get(crtc, pipe_config);
9351         } else {
9352                 pipe_config->pixel_multiplier = 1;
9353         }
9354
9355         intel_get_pipe_timings(crtc, pipe_config);
9356
9357         ironlake_get_pfit_config(crtc, pipe_config);
9358
9359         return true;
9360 }
9361
9362 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
9363 {
9364         struct drm_device *dev = dev_priv->dev;
9365         struct intel_crtc *crtc;
9366
9367         for_each_intel_crtc(dev, crtc)
9368                 I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
9369                      pipe_name(crtc->pipe));
9370
9371         I915_STATE_WARN(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on\n");
9372         I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n");
9373         I915_STATE_WARN(I915_READ(WRPLL_CTL1) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
9374         I915_STATE_WARN(I915_READ(WRPLL_CTL2) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
9375         I915_STATE_WARN(I915_READ(PCH_PP_STATUS) & PP_ON, "Panel power on\n");
9376         I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
9377              "CPU PWM1 enabled\n");
9378         if (IS_HASWELL(dev))
9379                 I915_STATE_WARN(I915_READ(HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
9380                      "CPU PWM2 enabled\n");
9381         I915_STATE_WARN(I915_READ(BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
9382              "PCH PWM1 enabled\n");
9383         I915_STATE_WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
9384              "Utility pin enabled\n");
9385         I915_STATE_WARN(I915_READ(PCH_GTC_CTL) & PCH_GTC_ENABLE, "PCH GTC enabled\n");
9386
9387         /*
9388          * In theory we can still leave IRQs enabled, as long as only the HPD
9389          * interrupts remain enabled. We used to check for that, but since it's
9390          * gen-specific and since we only disable LCPLL after we fully disable
9391          * the interrupts, the check below should be enough.
9392          */
9393         I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n");
9394 }
9395
9396 static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
9397 {
9398         struct drm_device *dev = dev_priv->dev;
9399
9400         if (IS_HASWELL(dev))
9401                 return I915_READ(D_COMP_HSW);
9402         else
9403                 return I915_READ(D_COMP_BDW);
9404 }
9405
9406 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
9407 {
9408         struct drm_device *dev = dev_priv->dev;
9409
9410         if (IS_HASWELL(dev)) {
9411                 mutex_lock(&dev_priv->rps.hw_lock);
9412                 if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP,
9413                                             val))
9414                         DRM_ERROR("Failed to write to D_COMP\n");
9415                 mutex_unlock(&dev_priv->rps.hw_lock);
9416         } else {
9417                 I915_WRITE(D_COMP_BDW, val);
9418                 POSTING_READ(D_COMP_BDW);
9419         }
9420 }
9421
9422 /*
9423  * This function implements pieces of two sequences from BSpec:
9424  * - Sequence for display software to disable LCPLL
9425  * - Sequence for display software to allow package C8+
9426  * The steps implemented here are just the steps that actually touch the LCPLL
9427  * register. Callers should take care of disabling all the display engine
9428  * functions, doing the mode unset, fixing interrupts, etc.
9429  */
9430 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
9431                               bool switch_to_fclk, bool allow_power_down)
9432 {
9433         uint32_t val;
9434
9435         assert_can_disable_lcpll(dev_priv);
9436
9437         val = I915_READ(LCPLL_CTL);
9438
9439         if (switch_to_fclk) {
9440                 val |= LCPLL_CD_SOURCE_FCLK;
9441                 I915_WRITE(LCPLL_CTL, val);
9442
9443                 if (wait_for_atomic_us(I915_READ(LCPLL_CTL) &
9444                                        LCPLL_CD_SOURCE_FCLK_DONE, 1))
9445                         DRM_ERROR("Switching to FCLK failed\n");
9446
9447                 val = I915_READ(LCPLL_CTL);
9448         }
9449
9450         val |= LCPLL_PLL_DISABLE;
9451         I915_WRITE(LCPLL_CTL, val);
9452         POSTING_READ(LCPLL_CTL);
9453
9454         if (wait_for((I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK) == 0, 1))
9455                 DRM_ERROR("LCPLL still locked\n");
9456
9457         val = hsw_read_dcomp(dev_priv);
9458         val |= D_COMP_COMP_DISABLE;
9459         hsw_write_dcomp(dev_priv, val);
9460         ndelay(100);
9461
9462         if (wait_for((hsw_read_dcomp(dev_priv) & D_COMP_RCOMP_IN_PROGRESS) == 0,
9463                      1))
9464                 DRM_ERROR("D_COMP RCOMP still in progress\n");
9465
9466         if (allow_power_down) {
9467                 val = I915_READ(LCPLL_CTL);
9468                 val |= LCPLL_POWER_DOWN_ALLOW;
9469                 I915_WRITE(LCPLL_CTL, val);
9470                 POSTING_READ(LCPLL_CTL);
9471         }
9472 }
9473
9474 /*
9475  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
9476  * source.
9477  */
9478 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
9479 {
9480         uint32_t val;
9481
9482         val = I915_READ(LCPLL_CTL);
9483
9484         if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
9485                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
9486                 return;
9487
9488         /*
9489          * Make sure we're not on PC8 state before disabling PC8, otherwise
9490          * we'll hang the machine. To prevent PC8 state, just enable force_wake.
9491          */
9492         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
9493
9494         if (val & LCPLL_POWER_DOWN_ALLOW) {
9495                 val &= ~LCPLL_POWER_DOWN_ALLOW;
9496                 I915_WRITE(LCPLL_CTL, val);
9497                 POSTING_READ(LCPLL_CTL);
9498         }
9499
9500         val = hsw_read_dcomp(dev_priv);
9501         val |= D_COMP_COMP_FORCE;
9502         val &= ~D_COMP_COMP_DISABLE;
9503         hsw_write_dcomp(dev_priv, val);
9504
9505         val = I915_READ(LCPLL_CTL);
9506         val &= ~LCPLL_PLL_DISABLE;
9507         I915_WRITE(LCPLL_CTL, val);
9508
9509         if (wait_for(I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK, 5))
9510                 DRM_ERROR("LCPLL not locked yet\n");
9511
9512         if (val & LCPLL_CD_SOURCE_FCLK) {
9513                 val = I915_READ(LCPLL_CTL);
9514                 val &= ~LCPLL_CD_SOURCE_FCLK;
9515                 I915_WRITE(LCPLL_CTL, val);
9516
9517                 if (wait_for_atomic_us((I915_READ(LCPLL_CTL) &
9518                                         LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9519                         DRM_ERROR("Switching back to LCPLL failed\n");
9520         }
9521
9522         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
9523         intel_update_cdclk(dev_priv->dev);
9524 }
9525
9526 /*
9527  * Package states C8 and deeper are really deep PC states that can only be
9528  * reached when all the devices on the system allow it, so even if the graphics
9529  * device allows PC8+, it doesn't mean the system will actually get to these
9530  * states. Our driver only allows PC8+ when going into runtime PM.
9531  *
9532  * The requirements for PC8+ are that all the outputs are disabled, the power
9533  * well is disabled and most interrupts are disabled, and these are also
9534  * requirements for runtime PM. When these conditions are met, we manually do
9535  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
9536  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
9537  * hang the machine.
9538  *
9539  * When we really reach PC8 or deeper states (not just when we allow it) we lose
9540  * the state of some registers, so when we come back from PC8+ we need to
9541  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
9542  * need to take care of the registers kept by RC6. Notice that this happens even
9543  * if we don't put the device in PCI D3 state (which is what currently happens
9544  * because of the runtime PM support).
9545  *
9546  * For more, read "Display Sequences for Package C8" on the hardware
9547  * documentation.
9548  */
9549 void hsw_enable_pc8(struct drm_i915_private *dev_priv)
9550 {
9551         struct drm_device *dev = dev_priv->dev;
9552         uint32_t val;
9553
9554         DRM_DEBUG_KMS("Enabling package C8+\n");
9555
9556         if (HAS_PCH_LPT_LP(dev)) {
9557                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9558                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
9559                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9560         }
9561
9562         lpt_disable_clkout_dp(dev);
9563         hsw_disable_lcpll(dev_priv, true, true);
9564 }
9565
9566 void hsw_disable_pc8(struct drm_i915_private *dev_priv)
9567 {
9568         struct drm_device *dev = dev_priv->dev;
9569         uint32_t val;
9570
9571         DRM_DEBUG_KMS("Disabling package C8+\n");
9572
9573         hsw_restore_lcpll(dev_priv);
9574         lpt_init_pch_refclk(dev);
9575
9576         if (HAS_PCH_LPT_LP(dev)) {
9577                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9578                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
9579                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9580         }
9581
9582         intel_prepare_ddi(dev);
9583 }
9584
9585 static void broxton_modeset_commit_cdclk(struct drm_atomic_state *old_state)
9586 {
9587         struct drm_device *dev = old_state->dev;
9588         unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
9589
9590         broxton_set_cdclk(dev, req_cdclk);
9591 }
9592
9593 /* compute the max rate for new configuration */
9594 static int ilk_max_pixel_rate(struct drm_atomic_state *state)
9595 {
9596         struct intel_crtc *intel_crtc;
9597         struct intel_crtc_state *crtc_state;
9598         int max_pixel_rate = 0;
9599
9600         for_each_intel_crtc(state->dev, intel_crtc) {
9601                 int pixel_rate;
9602
9603                 crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
9604                 if (IS_ERR(crtc_state))
9605                         return PTR_ERR(crtc_state);
9606
9607                 if (!crtc_state->base.enable)
9608                         continue;
9609
9610                 pixel_rate = ilk_pipe_pixel_rate(crtc_state);
9611
9612                 /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
9613                 if (IS_BROADWELL(state->dev) && crtc_state->ips_enabled)
9614                         pixel_rate = DIV_ROUND_UP(pixel_rate * 100, 95);
9615
9616                 max_pixel_rate = max(max_pixel_rate, pixel_rate);
9617         }
9618
9619         return max_pixel_rate;
9620 }
9621
9622 static void broadwell_set_cdclk(struct drm_device *dev, int cdclk)
9623 {
9624         struct drm_i915_private *dev_priv = dev->dev_private;
9625         uint32_t val, data;
9626         int ret;
9627
9628         if (WARN((I915_READ(LCPLL_CTL) &
9629                   (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
9630                    LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
9631                    LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
9632                    LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
9633                  "trying to change cdclk frequency with cdclk not enabled\n"))
9634                 return;
9635
9636         mutex_lock(&dev_priv->rps.hw_lock);
9637         ret = sandybridge_pcode_write(dev_priv,
9638                                       BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
9639         mutex_unlock(&dev_priv->rps.hw_lock);
9640         if (ret) {
9641                 DRM_ERROR("failed to inform pcode about cdclk change\n");
9642                 return;
9643         }
9644
9645         val = I915_READ(LCPLL_CTL);
9646         val |= LCPLL_CD_SOURCE_FCLK;
9647         I915_WRITE(LCPLL_CTL, val);
9648
9649         if (wait_for_atomic_us(I915_READ(LCPLL_CTL) &
9650                                LCPLL_CD_SOURCE_FCLK_DONE, 1))
9651                 DRM_ERROR("Switching to FCLK failed\n");
9652
9653         val = I915_READ(LCPLL_CTL);
9654         val &= ~LCPLL_CLK_FREQ_MASK;
9655
9656         switch (cdclk) {
9657         case 450000:
9658                 val |= LCPLL_CLK_FREQ_450;
9659                 data = 0;
9660                 break;
9661         case 540000:
9662                 val |= LCPLL_CLK_FREQ_54O_BDW;
9663                 data = 1;
9664                 break;
9665         case 337500:
9666                 val |= LCPLL_CLK_FREQ_337_5_BDW;
9667                 data = 2;
9668                 break;
9669         case 675000:
9670                 val |= LCPLL_CLK_FREQ_675_BDW;
9671                 data = 3;
9672                 break;
9673         default:
9674                 WARN(1, "invalid cdclk frequency\n");
9675                 return;
9676         }
9677
9678         I915_WRITE(LCPLL_CTL, val);
9679
9680         val = I915_READ(LCPLL_CTL);
9681         val &= ~LCPLL_CD_SOURCE_FCLK;
9682         I915_WRITE(LCPLL_CTL, val);
9683
9684         if (wait_for_atomic_us((I915_READ(LCPLL_CTL) &
9685                                 LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9686                 DRM_ERROR("Switching back to LCPLL failed\n");
9687
9688         mutex_lock(&dev_priv->rps.hw_lock);
9689         sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, data);
9690         mutex_unlock(&dev_priv->rps.hw_lock);
9691
9692         I915_WRITE(CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
9693
9694         intel_update_cdclk(dev);
9695
9696         WARN(cdclk != dev_priv->cdclk_freq,
9697              "cdclk requested %d kHz but got %d kHz\n",
9698              cdclk, dev_priv->cdclk_freq);
9699 }
9700
9701 static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state)
9702 {
9703         struct drm_i915_private *dev_priv = to_i915(state->dev);
9704         int max_pixclk = ilk_max_pixel_rate(state);
9705         int cdclk;
9706
9707         /*
9708          * FIXME should also account for plane ratio
9709          * once 64bpp pixel formats are supported.
9710          */
9711         if (max_pixclk > 540000)
9712                 cdclk = 675000;
9713         else if (max_pixclk > 450000)
9714                 cdclk = 540000;
9715         else if (max_pixclk > 337500)
9716                 cdclk = 450000;
9717         else
9718                 cdclk = 337500;
9719
9720         /*
9721          * FIXME move the cdclk caclulation to
9722          * compute_config() so we can fail gracegully.
9723          */
9724         if (cdclk > dev_priv->max_cdclk_freq) {
9725                 DRM_ERROR("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
9726                           cdclk, dev_priv->max_cdclk_freq);
9727                 cdclk = dev_priv->max_cdclk_freq;
9728         }
9729
9730         to_intel_atomic_state(state)->cdclk = cdclk;
9731
9732         return 0;
9733 }
9734
9735 static void broadwell_modeset_commit_cdclk(struct drm_atomic_state *old_state)
9736 {
9737         struct drm_device *dev = old_state->dev;
9738         unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
9739
9740         broadwell_set_cdclk(dev, req_cdclk);
9741 }
9742
9743 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
9744                                       struct intel_crtc_state *crtc_state)
9745 {
9746         if (!intel_ddi_pll_select(crtc, crtc_state))
9747                 return -EINVAL;
9748
9749         crtc->lowfreq_avail = false;
9750
9751         return 0;
9752 }
9753
9754 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
9755                                 enum port port,
9756                                 struct intel_crtc_state *pipe_config)
9757 {
9758         switch (port) {
9759         case PORT_A:
9760                 pipe_config->ddi_pll_sel = SKL_DPLL0;
9761                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL1;
9762                 break;
9763         case PORT_B:
9764                 pipe_config->ddi_pll_sel = SKL_DPLL1;
9765                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL2;
9766                 break;
9767         case PORT_C:
9768                 pipe_config->ddi_pll_sel = SKL_DPLL2;
9769                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL3;
9770                 break;
9771         default:
9772                 DRM_ERROR("Incorrect port type\n");
9773         }
9774 }
9775
9776 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
9777                                 enum port port,
9778                                 struct intel_crtc_state *pipe_config)
9779 {
9780         u32 temp, dpll_ctl1;
9781
9782         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
9783         pipe_config->ddi_pll_sel = temp >> (port * 3 + 1);
9784
9785         switch (pipe_config->ddi_pll_sel) {
9786         case SKL_DPLL0:
9787                 /*
9788                  * On SKL the eDP DPLL (DPLL0 as we don't use SSC) is not part
9789                  * of the shared DPLL framework and thus needs to be read out
9790                  * separately
9791                  */
9792                 dpll_ctl1 = I915_READ(DPLL_CTRL1);
9793                 pipe_config->dpll_hw_state.ctrl1 = dpll_ctl1 & 0x3f;
9794                 break;
9795         case SKL_DPLL1:
9796                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL1;
9797                 break;
9798         case SKL_DPLL2:
9799                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL2;
9800                 break;
9801         case SKL_DPLL3:
9802                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL3;
9803                 break;
9804         }
9805 }
9806
9807 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
9808                                 enum port port,
9809                                 struct intel_crtc_state *pipe_config)
9810 {
9811         pipe_config->ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
9812
9813         switch (pipe_config->ddi_pll_sel) {
9814         case PORT_CLK_SEL_WRPLL1:
9815                 pipe_config->shared_dpll = DPLL_ID_WRPLL1;
9816                 break;
9817         case PORT_CLK_SEL_WRPLL2:
9818                 pipe_config->shared_dpll = DPLL_ID_WRPLL2;
9819                 break;
9820         case PORT_CLK_SEL_SPLL:
9821                 pipe_config->shared_dpll = DPLL_ID_SPLL;
9822         }
9823 }
9824
9825 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
9826                                        struct intel_crtc_state *pipe_config)
9827 {
9828         struct drm_device *dev = crtc->base.dev;
9829         struct drm_i915_private *dev_priv = dev->dev_private;
9830         struct intel_shared_dpll *pll;
9831         enum port port;
9832         uint32_t tmp;
9833
9834         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
9835
9836         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
9837
9838         if (IS_SKYLAKE(dev))
9839                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
9840         else if (IS_BROXTON(dev))
9841                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
9842         else
9843                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
9844
9845         if (pipe_config->shared_dpll >= 0) {
9846                 pll = &dev_priv->shared_dplls[pipe_config->shared_dpll];
9847
9848                 WARN_ON(!pll->get_hw_state(dev_priv, pll,
9849                                            &pipe_config->dpll_hw_state));
9850         }
9851
9852         /*
9853          * Haswell has only FDI/PCH transcoder A. It is which is connected to
9854          * DDI E. So just check whether this pipe is wired to DDI E and whether
9855          * the PCH transcoder is on.
9856          */
9857         if (INTEL_INFO(dev)->gen < 9 &&
9858             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
9859                 pipe_config->has_pch_encoder = true;
9860
9861                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
9862                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9863                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9864
9865                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9866         }
9867 }
9868
9869 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
9870                                     struct intel_crtc_state *pipe_config)
9871 {
9872         struct drm_device *dev = crtc->base.dev;
9873         struct drm_i915_private *dev_priv = dev->dev_private;
9874         enum intel_display_power_domain pfit_domain;
9875         uint32_t tmp;
9876
9877         if (!intel_display_power_is_enabled(dev_priv,
9878                                          POWER_DOMAIN_PIPE(crtc->pipe)))
9879                 return false;
9880
9881         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9882         pipe_config->shared_dpll = DPLL_ID_PRIVATE;
9883
9884         tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP));
9885         if (tmp & TRANS_DDI_FUNC_ENABLE) {
9886                 enum pipe trans_edp_pipe;
9887                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
9888                 default:
9889                         WARN(1, "unknown pipe linked to edp transcoder\n");
9890                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
9891                 case TRANS_DDI_EDP_INPUT_A_ON:
9892                         trans_edp_pipe = PIPE_A;
9893                         break;
9894                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
9895                         trans_edp_pipe = PIPE_B;
9896                         break;
9897                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
9898                         trans_edp_pipe = PIPE_C;
9899                         break;
9900                 }
9901
9902                 if (trans_edp_pipe == crtc->pipe)
9903                         pipe_config->cpu_transcoder = TRANSCODER_EDP;
9904         }
9905
9906         if (!intel_display_power_is_enabled(dev_priv,
9907                         POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder)))
9908                 return false;
9909
9910         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
9911         if (!(tmp & PIPECONF_ENABLE))
9912                 return false;
9913
9914         haswell_get_ddi_port_state(crtc, pipe_config);
9915
9916         intel_get_pipe_timings(crtc, pipe_config);
9917
9918         if (INTEL_INFO(dev)->gen >= 9) {
9919                 skl_init_scalers(dev, crtc, pipe_config);
9920         }
9921
9922         pfit_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
9923
9924         if (INTEL_INFO(dev)->gen >= 9) {
9925                 pipe_config->scaler_state.scaler_id = -1;
9926                 pipe_config->scaler_state.scaler_users &= ~(1 << SKL_CRTC_INDEX);
9927         }
9928
9929         if (intel_display_power_is_enabled(dev_priv, pfit_domain)) {
9930                 if (INTEL_INFO(dev)->gen >= 9)
9931                         skylake_get_pfit_config(crtc, pipe_config);
9932                 else
9933                         ironlake_get_pfit_config(crtc, pipe_config);
9934         }
9935
9936         if (IS_HASWELL(dev))
9937                 pipe_config->ips_enabled = hsw_crtc_supports_ips(crtc) &&
9938                         (I915_READ(IPS_CTL) & IPS_ENABLE);
9939
9940         if (pipe_config->cpu_transcoder != TRANSCODER_EDP) {
9941                 pipe_config->pixel_multiplier =
9942                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
9943         } else {
9944                 pipe_config->pixel_multiplier = 1;
9945         }
9946
9947         return true;
9948 }
9949
9950 static void i845_update_cursor(struct drm_crtc *crtc, u32 base, bool on)
9951 {
9952         struct drm_device *dev = crtc->dev;
9953         struct drm_i915_private *dev_priv = dev->dev_private;
9954         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
9955         uint32_t cntl = 0, size = 0;
9956
9957         if (on) {
9958                 unsigned int width = intel_crtc->base.cursor->state->crtc_w;
9959                 unsigned int height = intel_crtc->base.cursor->state->crtc_h;
9960                 unsigned int stride = roundup_pow_of_two(width) * 4;
9961
9962                 switch (stride) {
9963                 default:
9964                         WARN_ONCE(1, "Invalid cursor width/stride, width=%u, stride=%u\n",
9965                                   width, stride);
9966                         stride = 256;
9967                         /* fallthrough */
9968                 case 256:
9969                 case 512:
9970                 case 1024:
9971                 case 2048:
9972                         break;
9973                 }
9974
9975                 cntl |= CURSOR_ENABLE |
9976                         CURSOR_GAMMA_ENABLE |
9977                         CURSOR_FORMAT_ARGB |
9978                         CURSOR_STRIDE(stride);
9979
9980                 size = (height << 12) | width;
9981         }
9982
9983         if (intel_crtc->cursor_cntl != 0 &&
9984             (intel_crtc->cursor_base != base ||
9985              intel_crtc->cursor_size != size ||
9986              intel_crtc->cursor_cntl != cntl)) {
9987                 /* On these chipsets we can only modify the base/size/stride
9988                  * whilst the cursor is disabled.
9989                  */
9990                 I915_WRITE(CURCNTR(PIPE_A), 0);
9991                 POSTING_READ(CURCNTR(PIPE_A));
9992                 intel_crtc->cursor_cntl = 0;
9993         }
9994
9995         if (intel_crtc->cursor_base != base) {
9996                 I915_WRITE(CURBASE(PIPE_A), base);
9997                 intel_crtc->cursor_base = base;
9998         }
9999
10000         if (intel_crtc->cursor_size != size) {
10001                 I915_WRITE(CURSIZE, size);
10002                 intel_crtc->cursor_size = size;
10003         }
10004
10005         if (intel_crtc->cursor_cntl != cntl) {
10006                 I915_WRITE(CURCNTR(PIPE_A), cntl);
10007                 POSTING_READ(CURCNTR(PIPE_A));
10008                 intel_crtc->cursor_cntl = cntl;
10009         }
10010 }
10011
10012 static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base, bool on)
10013 {
10014         struct drm_device *dev = crtc->dev;
10015         struct drm_i915_private *dev_priv = dev->dev_private;
10016         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10017         int pipe = intel_crtc->pipe;
10018         uint32_t cntl = 0;
10019
10020         if (on) {
10021                 cntl = MCURSOR_GAMMA_ENABLE;
10022                 switch (intel_crtc->base.cursor->state->crtc_w) {
10023                         case 64:
10024                                 cntl |= CURSOR_MODE_64_ARGB_AX;
10025                                 break;
10026                         case 128:
10027                                 cntl |= CURSOR_MODE_128_ARGB_AX;
10028                                 break;
10029                         case 256:
10030                                 cntl |= CURSOR_MODE_256_ARGB_AX;
10031                                 break;
10032                         default:
10033                                 MISSING_CASE(intel_crtc->base.cursor->state->crtc_w);
10034                                 return;
10035                 }
10036                 cntl |= pipe << 28; /* Connect to correct pipe */
10037
10038                 if (HAS_DDI(dev))
10039                         cntl |= CURSOR_PIPE_CSC_ENABLE;
10040         }
10041
10042         if (crtc->cursor->state->rotation == BIT(DRM_ROTATE_180))
10043                 cntl |= CURSOR_ROTATE_180;
10044
10045         if (intel_crtc->cursor_cntl != cntl) {
10046                 I915_WRITE(CURCNTR(pipe), cntl);
10047                 POSTING_READ(CURCNTR(pipe));
10048                 intel_crtc->cursor_cntl = cntl;
10049         }
10050
10051         /* and commit changes on next vblank */
10052         I915_WRITE(CURBASE(pipe), base);
10053         POSTING_READ(CURBASE(pipe));
10054
10055         intel_crtc->cursor_base = base;
10056 }
10057
10058 /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
10059 static void intel_crtc_update_cursor(struct drm_crtc *crtc,
10060                                      bool on)
10061 {
10062         struct drm_device *dev = crtc->dev;
10063         struct drm_i915_private *dev_priv = dev->dev_private;
10064         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10065         int pipe = intel_crtc->pipe;
10066         struct drm_plane_state *cursor_state = crtc->cursor->state;
10067         int x = cursor_state->crtc_x;
10068         int y = cursor_state->crtc_y;
10069         u32 base = 0, pos = 0;
10070
10071         base = intel_crtc->cursor_addr;
10072
10073         if (x >= intel_crtc->config->pipe_src_w)
10074                 on = false;
10075
10076         if (y >= intel_crtc->config->pipe_src_h)
10077                 on = false;
10078
10079         if (x < 0) {
10080                 if (x + cursor_state->crtc_w <= 0)
10081                         on = false;
10082
10083                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
10084                 x = -x;
10085         }
10086         pos |= x << CURSOR_X_SHIFT;
10087
10088         if (y < 0) {
10089                 if (y + cursor_state->crtc_h <= 0)
10090                         on = false;
10091
10092                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
10093                 y = -y;
10094         }
10095         pos |= y << CURSOR_Y_SHIFT;
10096
10097         I915_WRITE(CURPOS(pipe), pos);
10098
10099         /* ILK+ do this automagically */
10100         if (HAS_GMCH_DISPLAY(dev) &&
10101             crtc->cursor->state->rotation == BIT(DRM_ROTATE_180)) {
10102                 base += (cursor_state->crtc_h *
10103                          cursor_state->crtc_w - 1) * 4;
10104         }
10105
10106         if (IS_845G(dev) || IS_I865G(dev))
10107                 i845_update_cursor(crtc, base, on);
10108         else
10109                 i9xx_update_cursor(crtc, base, on);
10110 }
10111
10112 static bool cursor_size_ok(struct drm_device *dev,
10113                            uint32_t width, uint32_t height)
10114 {
10115         if (width == 0 || height == 0)
10116                 return false;
10117
10118         /*
10119          * 845g/865g are special in that they are only limited by
10120          * the width of their cursors, the height is arbitrary up to
10121          * the precision of the register. Everything else requires
10122          * square cursors, limited to a few power-of-two sizes.
10123          */
10124         if (IS_845G(dev) || IS_I865G(dev)) {
10125                 if ((width & 63) != 0)
10126                         return false;
10127
10128                 if (width > (IS_845G(dev) ? 64 : 512))
10129                         return false;
10130
10131                 if (height > 1023)
10132                         return false;
10133         } else {
10134                 switch (width | height) {
10135                 case 256:
10136                 case 128:
10137                         if (IS_GEN2(dev))
10138                                 return false;
10139                 case 64:
10140                         break;
10141                 default:
10142                         return false;
10143                 }
10144         }
10145
10146         return true;
10147 }
10148
10149 static void intel_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
10150                                  u16 *blue, uint32_t start, uint32_t size)
10151 {
10152         int end = (start + size > 256) ? 256 : start + size, i;
10153         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10154
10155         for (i = start; i < end; i++) {
10156                 intel_crtc->lut_r[i] = red[i] >> 8;
10157                 intel_crtc->lut_g[i] = green[i] >> 8;
10158                 intel_crtc->lut_b[i] = blue[i] >> 8;
10159         }
10160
10161         intel_crtc_load_lut(crtc);
10162 }
10163
10164 /* VESA 640x480x72Hz mode to set on the pipe */
10165 static struct drm_display_mode load_detect_mode = {
10166         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
10167                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
10168 };
10169
10170 struct drm_framebuffer *
10171 __intel_framebuffer_create(struct drm_device *dev,
10172                            struct drm_mode_fb_cmd2 *mode_cmd,
10173                            struct drm_i915_gem_object *obj)
10174 {
10175         struct intel_framebuffer *intel_fb;
10176         int ret;
10177
10178         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10179         if (!intel_fb) {
10180                 drm_gem_object_unreference(&obj->base);
10181                 return ERR_PTR(-ENOMEM);
10182         }
10183
10184         ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
10185         if (ret)
10186                 goto err;
10187
10188         return &intel_fb->base;
10189 err:
10190         drm_gem_object_unreference(&obj->base);
10191         kfree(intel_fb);
10192
10193         return ERR_PTR(ret);
10194 }
10195
10196 static struct drm_framebuffer *
10197 intel_framebuffer_create(struct drm_device *dev,
10198                          struct drm_mode_fb_cmd2 *mode_cmd,
10199                          struct drm_i915_gem_object *obj)
10200 {
10201         struct drm_framebuffer *fb;
10202         int ret;
10203
10204         ret = i915_mutex_lock_interruptible(dev);
10205         if (ret)
10206                 return ERR_PTR(ret);
10207         fb = __intel_framebuffer_create(dev, mode_cmd, obj);
10208         mutex_unlock(&dev->struct_mutex);
10209
10210         return fb;
10211 }
10212
10213 static u32
10214 intel_framebuffer_pitch_for_width(int width, int bpp)
10215 {
10216         u32 pitch = DIV_ROUND_UP(width * bpp, 8);
10217         return ALIGN(pitch, 64);
10218 }
10219
10220 static u32
10221 intel_framebuffer_size_for_mode(struct drm_display_mode *mode, int bpp)
10222 {
10223         u32 pitch = intel_framebuffer_pitch_for_width(mode->hdisplay, bpp);
10224         return PAGE_ALIGN(pitch * mode->vdisplay);
10225 }
10226
10227 static struct drm_framebuffer *
10228 intel_framebuffer_create_for_mode(struct drm_device *dev,
10229                                   struct drm_display_mode *mode,
10230                                   int depth, int bpp)
10231 {
10232         struct drm_i915_gem_object *obj;
10233         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
10234
10235         obj = i915_gem_alloc_object(dev,
10236                                     intel_framebuffer_size_for_mode(mode, bpp));
10237         if (obj == NULL)
10238                 return ERR_PTR(-ENOMEM);
10239
10240         mode_cmd.width = mode->hdisplay;
10241         mode_cmd.height = mode->vdisplay;
10242         mode_cmd.pitches[0] = intel_framebuffer_pitch_for_width(mode_cmd.width,
10243                                                                 bpp);
10244         mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth);
10245
10246         return intel_framebuffer_create(dev, &mode_cmd, obj);
10247 }
10248
10249 static struct drm_framebuffer *
10250 mode_fits_in_fbdev(struct drm_device *dev,
10251                    struct drm_display_mode *mode)
10252 {
10253 #ifdef CONFIG_DRM_FBDEV_EMULATION
10254         struct drm_i915_private *dev_priv = dev->dev_private;
10255         struct drm_i915_gem_object *obj;
10256         struct drm_framebuffer *fb;
10257
10258         if (!dev_priv->fbdev)
10259                 return NULL;
10260
10261         if (!dev_priv->fbdev->fb)
10262                 return NULL;
10263
10264         obj = dev_priv->fbdev->fb->obj;
10265         BUG_ON(!obj);
10266
10267         fb = &dev_priv->fbdev->fb->base;
10268         if (fb->pitches[0] < intel_framebuffer_pitch_for_width(mode->hdisplay,
10269                                                                fb->bits_per_pixel))
10270                 return NULL;
10271
10272         if (obj->base.size < mode->vdisplay * fb->pitches[0])
10273                 return NULL;
10274
10275         return fb;
10276 #else
10277         return NULL;
10278 #endif
10279 }
10280
10281 static int intel_modeset_setup_plane_state(struct drm_atomic_state *state,
10282                                            struct drm_crtc *crtc,
10283                                            struct drm_display_mode *mode,
10284                                            struct drm_framebuffer *fb,
10285                                            int x, int y)
10286 {
10287         struct drm_plane_state *plane_state;
10288         int hdisplay, vdisplay;
10289         int ret;
10290
10291         plane_state = drm_atomic_get_plane_state(state, crtc->primary);
10292         if (IS_ERR(plane_state))
10293                 return PTR_ERR(plane_state);
10294
10295         if (mode)
10296                 drm_crtc_get_hv_timing(mode, &hdisplay, &vdisplay);
10297         else
10298                 hdisplay = vdisplay = 0;
10299
10300         ret = drm_atomic_set_crtc_for_plane(plane_state, fb ? crtc : NULL);
10301         if (ret)
10302                 return ret;
10303         drm_atomic_set_fb_for_plane(plane_state, fb);
10304         plane_state->crtc_x = 0;
10305         plane_state->crtc_y = 0;
10306         plane_state->crtc_w = hdisplay;
10307         plane_state->crtc_h = vdisplay;
10308         plane_state->src_x = x << 16;
10309         plane_state->src_y = y << 16;
10310         plane_state->src_w = hdisplay << 16;
10311         plane_state->src_h = vdisplay << 16;
10312
10313         return 0;
10314 }
10315
10316 bool intel_get_load_detect_pipe(struct drm_connector *connector,
10317                                 struct drm_display_mode *mode,
10318                                 struct intel_load_detect_pipe *old,
10319                                 struct drm_modeset_acquire_ctx *ctx)
10320 {
10321         struct intel_crtc *intel_crtc;
10322         struct intel_encoder *intel_encoder =
10323                 intel_attached_encoder(connector);
10324         struct drm_crtc *possible_crtc;
10325         struct drm_encoder *encoder = &intel_encoder->base;
10326         struct drm_crtc *crtc = NULL;
10327         struct drm_device *dev = encoder->dev;
10328         struct drm_framebuffer *fb;
10329         struct drm_mode_config *config = &dev->mode_config;
10330         struct drm_atomic_state *state = NULL;
10331         struct drm_connector_state *connector_state;
10332         struct intel_crtc_state *crtc_state;
10333         int ret, i = -1;
10334
10335         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10336                       connector->base.id, connector->name,
10337                       encoder->base.id, encoder->name);
10338
10339 retry:
10340         ret = drm_modeset_lock(&config->connection_mutex, ctx);
10341         if (ret)
10342                 goto fail;
10343
10344         /*
10345          * Algorithm gets a little messy:
10346          *
10347          *   - if the connector already has an assigned crtc, use it (but make
10348          *     sure it's on first)
10349          *
10350          *   - try to find the first unused crtc that can drive this connector,
10351          *     and use that if we find one
10352          */
10353
10354         /* See if we already have a CRTC for this connector */
10355         if (encoder->crtc) {
10356                 crtc = encoder->crtc;
10357
10358                 ret = drm_modeset_lock(&crtc->mutex, ctx);
10359                 if (ret)
10360                         goto fail;
10361                 ret = drm_modeset_lock(&crtc->primary->mutex, ctx);
10362                 if (ret)
10363                         goto fail;
10364
10365                 old->dpms_mode = connector->dpms;
10366                 old->load_detect_temp = false;
10367
10368                 /* Make sure the crtc and connector are running */
10369                 if (connector->dpms != DRM_MODE_DPMS_ON)
10370                         connector->funcs->dpms(connector, DRM_MODE_DPMS_ON);
10371
10372                 return true;
10373         }
10374
10375         /* Find an unused one (if possible) */
10376         for_each_crtc(dev, possible_crtc) {
10377                 i++;
10378                 if (!(encoder->possible_crtcs & (1 << i)))
10379                         continue;
10380                 if (possible_crtc->state->enable)
10381                         continue;
10382
10383                 crtc = possible_crtc;
10384                 break;
10385         }
10386
10387         /*
10388          * If we didn't find an unused CRTC, don't use any.
10389          */
10390         if (!crtc) {
10391                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
10392                 goto fail;
10393         }
10394
10395         ret = drm_modeset_lock(&crtc->mutex, ctx);
10396         if (ret)
10397                 goto fail;
10398         ret = drm_modeset_lock(&crtc->primary->mutex, ctx);
10399         if (ret)
10400                 goto fail;
10401
10402         intel_crtc = to_intel_crtc(crtc);
10403         old->dpms_mode = connector->dpms;
10404         old->load_detect_temp = true;
10405         old->release_fb = NULL;
10406
10407         state = drm_atomic_state_alloc(dev);
10408         if (!state)
10409                 return false;
10410
10411         state->acquire_ctx = ctx;
10412
10413         connector_state = drm_atomic_get_connector_state(state, connector);
10414         if (IS_ERR(connector_state)) {
10415                 ret = PTR_ERR(connector_state);
10416                 goto fail;
10417         }
10418
10419         connector_state->crtc = crtc;
10420         connector_state->best_encoder = &intel_encoder->base;
10421
10422         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
10423         if (IS_ERR(crtc_state)) {
10424                 ret = PTR_ERR(crtc_state);
10425                 goto fail;
10426         }
10427
10428         crtc_state->base.active = crtc_state->base.enable = true;
10429
10430         if (!mode)
10431                 mode = &load_detect_mode;
10432
10433         /* We need a framebuffer large enough to accommodate all accesses
10434          * that the plane may generate whilst we perform load detection.
10435          * We can not rely on the fbcon either being present (we get called
10436          * during its initialisation to detect all boot displays, or it may
10437          * not even exist) or that it is large enough to satisfy the
10438          * requested mode.
10439          */
10440         fb = mode_fits_in_fbdev(dev, mode);
10441         if (fb == NULL) {
10442                 DRM_DEBUG_KMS("creating tmp fb for load-detection\n");
10443                 fb = intel_framebuffer_create_for_mode(dev, mode, 24, 32);
10444                 old->release_fb = fb;
10445         } else
10446                 DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
10447         if (IS_ERR(fb)) {
10448                 DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
10449                 goto fail;
10450         }
10451
10452         ret = intel_modeset_setup_plane_state(state, crtc, mode, fb, 0, 0);
10453         if (ret)
10454                 goto fail;
10455
10456         drm_mode_copy(&crtc_state->base.mode, mode);
10457
10458         if (drm_atomic_commit(state)) {
10459                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
10460                 if (old->release_fb)
10461                         old->release_fb->funcs->destroy(old->release_fb);
10462                 goto fail;
10463         }
10464         crtc->primary->crtc = crtc;
10465
10466         /* let the connector get through one full cycle before testing */
10467         intel_wait_for_vblank(dev, intel_crtc->pipe);
10468         return true;
10469
10470 fail:
10471         drm_atomic_state_free(state);
10472         state = NULL;
10473
10474         if (ret == -EDEADLK) {
10475                 drm_modeset_backoff(ctx);
10476                 goto retry;
10477         }
10478
10479         return false;
10480 }
10481
10482 void intel_release_load_detect_pipe(struct drm_connector *connector,
10483                                     struct intel_load_detect_pipe *old,
10484                                     struct drm_modeset_acquire_ctx *ctx)
10485 {
10486         struct drm_device *dev = connector->dev;
10487         struct intel_encoder *intel_encoder =
10488                 intel_attached_encoder(connector);
10489         struct drm_encoder *encoder = &intel_encoder->base;
10490         struct drm_crtc *crtc = encoder->crtc;
10491         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10492         struct drm_atomic_state *state;
10493         struct drm_connector_state *connector_state;
10494         struct intel_crtc_state *crtc_state;
10495         int ret;
10496
10497         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10498                       connector->base.id, connector->name,
10499                       encoder->base.id, encoder->name);
10500
10501         if (old->load_detect_temp) {
10502                 state = drm_atomic_state_alloc(dev);
10503                 if (!state)
10504                         goto fail;
10505
10506                 state->acquire_ctx = ctx;
10507
10508                 connector_state = drm_atomic_get_connector_state(state, connector);
10509                 if (IS_ERR(connector_state))
10510                         goto fail;
10511
10512                 crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
10513                 if (IS_ERR(crtc_state))
10514                         goto fail;
10515
10516                 connector_state->best_encoder = NULL;
10517                 connector_state->crtc = NULL;
10518
10519                 crtc_state->base.enable = crtc_state->base.active = false;
10520
10521                 ret = intel_modeset_setup_plane_state(state, crtc, NULL, NULL,
10522                                                       0, 0);
10523                 if (ret)
10524                         goto fail;
10525
10526                 ret = drm_atomic_commit(state);
10527                 if (ret)
10528                         goto fail;
10529
10530                 if (old->release_fb) {
10531                         drm_framebuffer_unregister_private(old->release_fb);
10532                         drm_framebuffer_unreference(old->release_fb);
10533                 }
10534
10535                 return;
10536         }
10537
10538         /* Switch crtc and encoder back off if necessary */
10539         if (old->dpms_mode != DRM_MODE_DPMS_ON)
10540                 connector->funcs->dpms(connector, old->dpms_mode);
10541
10542         return;
10543 fail:
10544         DRM_DEBUG_KMS("Couldn't release load detect pipe.\n");
10545         drm_atomic_state_free(state);
10546 }
10547
10548 static int i9xx_pll_refclk(struct drm_device *dev,
10549                            const struct intel_crtc_state *pipe_config)
10550 {
10551         struct drm_i915_private *dev_priv = dev->dev_private;
10552         u32 dpll = pipe_config->dpll_hw_state.dpll;
10553
10554         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
10555                 return dev_priv->vbt.lvds_ssc_freq;
10556         else if (HAS_PCH_SPLIT(dev))
10557                 return 120000;
10558         else if (!IS_GEN2(dev))
10559                 return 96000;
10560         else
10561                 return 48000;
10562 }
10563
10564 /* Returns the clock of the currently programmed mode of the given pipe. */
10565 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
10566                                 struct intel_crtc_state *pipe_config)
10567 {
10568         struct drm_device *dev = crtc->base.dev;
10569         struct drm_i915_private *dev_priv = dev->dev_private;
10570         int pipe = pipe_config->cpu_transcoder;
10571         u32 dpll = pipe_config->dpll_hw_state.dpll;
10572         u32 fp;
10573         intel_clock_t clock;
10574         int port_clock;
10575         int refclk = i9xx_pll_refclk(dev, pipe_config);
10576
10577         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
10578                 fp = pipe_config->dpll_hw_state.fp0;
10579         else
10580                 fp = pipe_config->dpll_hw_state.fp1;
10581
10582         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
10583         if (IS_PINEVIEW(dev)) {
10584                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
10585                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
10586         } else {
10587                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
10588                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
10589         }
10590
10591         if (!IS_GEN2(dev)) {
10592                 if (IS_PINEVIEW(dev))
10593                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
10594                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
10595                 else
10596                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
10597                                DPLL_FPA01_P1_POST_DIV_SHIFT);
10598
10599                 switch (dpll & DPLL_MODE_MASK) {
10600                 case DPLLB_MODE_DAC_SERIAL:
10601                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
10602                                 5 : 10;
10603                         break;
10604                 case DPLLB_MODE_LVDS:
10605                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
10606                                 7 : 14;
10607                         break;
10608                 default:
10609                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
10610                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
10611                         return;
10612                 }
10613
10614                 if (IS_PINEVIEW(dev))
10615                         port_clock = pnv_calc_dpll_params(refclk, &clock);
10616                 else
10617                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
10618         } else {
10619                 u32 lvds = IS_I830(dev) ? 0 : I915_READ(LVDS);
10620                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
10621
10622                 if (is_lvds) {
10623                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
10624                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
10625
10626                         if (lvds & LVDS_CLKB_POWER_UP)
10627                                 clock.p2 = 7;
10628                         else
10629                                 clock.p2 = 14;
10630                 } else {
10631                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
10632                                 clock.p1 = 2;
10633                         else {
10634                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
10635                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
10636                         }
10637                         if (dpll & PLL_P2_DIVIDE_BY_4)
10638                                 clock.p2 = 4;
10639                         else
10640                                 clock.p2 = 2;
10641                 }
10642
10643                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
10644         }
10645
10646         /*
10647          * This value includes pixel_multiplier. We will use
10648          * port_clock to compute adjusted_mode.crtc_clock in the
10649          * encoder's get_config() function.
10650          */
10651         pipe_config->port_clock = port_clock;
10652 }
10653
10654 int intel_dotclock_calculate(int link_freq,
10655                              const struct intel_link_m_n *m_n)
10656 {
10657         /*
10658          * The calculation for the data clock is:
10659          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
10660          * But we want to avoid losing precison if possible, so:
10661          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
10662          *
10663          * and the link clock is simpler:
10664          * link_clock = (m * link_clock) / n
10665          */
10666
10667         if (!m_n->link_n)
10668                 return 0;
10669
10670         return div_u64((u64)m_n->link_m * link_freq, m_n->link_n);
10671 }
10672
10673 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
10674                                    struct intel_crtc_state *pipe_config)
10675 {
10676         struct drm_device *dev = crtc->base.dev;
10677
10678         /* read out port_clock from the DPLL */
10679         i9xx_crtc_clock_get(crtc, pipe_config);
10680
10681         /*
10682          * This value does not include pixel_multiplier.
10683          * We will check that port_clock and adjusted_mode.crtc_clock
10684          * agree once we know their relationship in the encoder's
10685          * get_config() function.
10686          */
10687         pipe_config->base.adjusted_mode.crtc_clock =
10688                 intel_dotclock_calculate(intel_fdi_link_freq(dev) * 10000,
10689                                          &pipe_config->fdi_m_n);
10690 }
10691
10692 /** Returns the currently programmed mode of the given pipe. */
10693 struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
10694                                              struct drm_crtc *crtc)
10695 {
10696         struct drm_i915_private *dev_priv = dev->dev_private;
10697         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10698         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
10699         struct drm_display_mode *mode;
10700         struct intel_crtc_state pipe_config;
10701         int htot = I915_READ(HTOTAL(cpu_transcoder));
10702         int hsync = I915_READ(HSYNC(cpu_transcoder));
10703         int vtot = I915_READ(VTOTAL(cpu_transcoder));
10704         int vsync = I915_READ(VSYNC(cpu_transcoder));
10705         enum pipe pipe = intel_crtc->pipe;
10706
10707         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
10708         if (!mode)
10709                 return NULL;
10710
10711         /*
10712          * Construct a pipe_config sufficient for getting the clock info
10713          * back out of crtc_clock_get.
10714          *
10715          * Note, if LVDS ever uses a non-1 pixel multiplier, we'll need
10716          * to use a real value here instead.
10717          */
10718         pipe_config.cpu_transcoder = (enum transcoder) pipe;
10719         pipe_config.pixel_multiplier = 1;
10720         pipe_config.dpll_hw_state.dpll = I915_READ(DPLL(pipe));
10721         pipe_config.dpll_hw_state.fp0 = I915_READ(FP0(pipe));
10722         pipe_config.dpll_hw_state.fp1 = I915_READ(FP1(pipe));
10723         i9xx_crtc_clock_get(intel_crtc, &pipe_config);
10724
10725         mode->clock = pipe_config.port_clock / pipe_config.pixel_multiplier;
10726         mode->hdisplay = (htot & 0xffff) + 1;
10727         mode->htotal = ((htot & 0xffff0000) >> 16) + 1;
10728         mode->hsync_start = (hsync & 0xffff) + 1;
10729         mode->hsync_end = ((hsync & 0xffff0000) >> 16) + 1;
10730         mode->vdisplay = (vtot & 0xffff) + 1;
10731         mode->vtotal = ((vtot & 0xffff0000) >> 16) + 1;
10732         mode->vsync_start = (vsync & 0xffff) + 1;
10733         mode->vsync_end = ((vsync & 0xffff0000) >> 16) + 1;
10734
10735         drm_mode_set_name(mode);
10736
10737         return mode;
10738 }
10739
10740 void intel_mark_busy(struct drm_device *dev)
10741 {
10742         struct drm_i915_private *dev_priv = dev->dev_private;
10743
10744         if (dev_priv->mm.busy)
10745                 return;
10746
10747         intel_runtime_pm_get(dev_priv);
10748         i915_update_gfx_val(dev_priv);
10749         if (INTEL_INFO(dev)->gen >= 6)
10750                 gen6_rps_busy(dev_priv);
10751         dev_priv->mm.busy = true;
10752 }
10753
10754 void intel_mark_idle(struct drm_device *dev)
10755 {
10756         struct drm_i915_private *dev_priv = dev->dev_private;
10757
10758         if (!dev_priv->mm.busy)
10759                 return;
10760
10761         dev_priv->mm.busy = false;
10762
10763         if (INTEL_INFO(dev)->gen >= 6)
10764                 gen6_rps_idle(dev->dev_private);
10765
10766         intel_runtime_pm_put(dev_priv);
10767 }
10768
10769 static void intel_crtc_destroy(struct drm_crtc *crtc)
10770 {
10771         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10772         struct drm_device *dev = crtc->dev;
10773         struct intel_unpin_work *work;
10774
10775         spin_lock_irq(&dev->event_lock);
10776         work = intel_crtc->unpin_work;
10777         intel_crtc->unpin_work = NULL;
10778         spin_unlock_irq(&dev->event_lock);
10779
10780         if (work) {
10781                 cancel_work_sync(&work->work);
10782                 kfree(work);
10783         }
10784
10785         drm_crtc_cleanup(crtc);
10786
10787         kfree(intel_crtc);
10788 }
10789
10790 static void intel_unpin_work_fn(struct work_struct *__work)
10791 {
10792         struct intel_unpin_work *work =
10793                 container_of(__work, struct intel_unpin_work, work);
10794         struct intel_crtc *crtc = to_intel_crtc(work->crtc);
10795         struct drm_device *dev = crtc->base.dev;
10796         struct drm_plane *primary = crtc->base.primary;
10797
10798         mutex_lock(&dev->struct_mutex);
10799         intel_unpin_fb_obj(work->old_fb, primary->state);
10800         drm_gem_object_unreference(&work->pending_flip_obj->base);
10801
10802         if (work->flip_queued_req)
10803                 i915_gem_request_assign(&work->flip_queued_req, NULL);
10804         mutex_unlock(&dev->struct_mutex);
10805
10806         intel_frontbuffer_flip_complete(dev, to_intel_plane(primary)->frontbuffer_bit);
10807         drm_framebuffer_unreference(work->old_fb);
10808
10809         BUG_ON(atomic_read(&crtc->unpin_work_count) == 0);
10810         atomic_dec(&crtc->unpin_work_count);
10811
10812         kfree(work);
10813 }
10814
10815 static void do_intel_finish_page_flip(struct drm_device *dev,
10816                                       struct drm_crtc *crtc)
10817 {
10818         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10819         struct intel_unpin_work *work;
10820         unsigned long flags;
10821
10822         /* Ignore early vblank irqs */
10823         if (intel_crtc == NULL)
10824                 return;
10825
10826         /*
10827          * This is called both by irq handlers and the reset code (to complete
10828          * lost pageflips) so needs the full irqsave spinlocks.
10829          */
10830         spin_lock_irqsave(&dev->event_lock, flags);
10831         work = intel_crtc->unpin_work;
10832
10833         /* Ensure we don't miss a work->pending update ... */
10834         smp_rmb();
10835
10836         if (work == NULL || atomic_read(&work->pending) < INTEL_FLIP_COMPLETE) {
10837                 spin_unlock_irqrestore(&dev->event_lock, flags);
10838                 return;
10839         }
10840
10841         page_flip_completed(intel_crtc);
10842
10843         spin_unlock_irqrestore(&dev->event_lock, flags);
10844 }
10845
10846 void intel_finish_page_flip(struct drm_device *dev, int pipe)
10847 {
10848         struct drm_i915_private *dev_priv = dev->dev_private;
10849         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
10850
10851         do_intel_finish_page_flip(dev, crtc);
10852 }
10853
10854 void intel_finish_page_flip_plane(struct drm_device *dev, int plane)
10855 {
10856         struct drm_i915_private *dev_priv = dev->dev_private;
10857         struct drm_crtc *crtc = dev_priv->plane_to_crtc_mapping[plane];
10858
10859         do_intel_finish_page_flip(dev, crtc);
10860 }
10861
10862 /* Is 'a' after or equal to 'b'? */
10863 static bool g4x_flip_count_after_eq(u32 a, u32 b)
10864 {
10865         return !((a - b) & 0x80000000);
10866 }
10867
10868 static bool page_flip_finished(struct intel_crtc *crtc)
10869 {
10870         struct drm_device *dev = crtc->base.dev;
10871         struct drm_i915_private *dev_priv = dev->dev_private;
10872
10873         if (i915_reset_in_progress(&dev_priv->gpu_error) ||
10874             crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
10875                 return true;
10876
10877         /*
10878          * The relevant registers doen't exist on pre-ctg.
10879          * As the flip done interrupt doesn't trigger for mmio
10880          * flips on gmch platforms, a flip count check isn't
10881          * really needed there. But since ctg has the registers,
10882          * include it in the check anyway.
10883          */
10884         if (INTEL_INFO(dev)->gen < 5 && !IS_G4X(dev))
10885                 return true;
10886
10887         /*
10888          * A DSPSURFLIVE check isn't enough in case the mmio and CS flips
10889          * used the same base address. In that case the mmio flip might
10890          * have completed, but the CS hasn't even executed the flip yet.
10891          *
10892          * A flip count check isn't enough as the CS might have updated
10893          * the base address just after start of vblank, but before we
10894          * managed to process the interrupt. This means we'd complete the
10895          * CS flip too soon.
10896          *
10897          * Combining both checks should get us a good enough result. It may
10898          * still happen that the CS flip has been executed, but has not
10899          * yet actually completed. But in case the base address is the same
10900          * anyway, we don't really care.
10901          */
10902         return (I915_READ(DSPSURFLIVE(crtc->plane)) & ~0xfff) ==
10903                 crtc->unpin_work->gtt_offset &&
10904                 g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_G4X(crtc->pipe)),
10905                                     crtc->unpin_work->flip_count);
10906 }
10907
10908 void intel_prepare_page_flip(struct drm_device *dev, int plane)
10909 {
10910         struct drm_i915_private *dev_priv = dev->dev_private;
10911         struct intel_crtc *intel_crtc =
10912                 to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]);
10913         unsigned long flags;
10914
10915
10916         /*
10917          * This is called both by irq handlers and the reset code (to complete
10918          * lost pageflips) so needs the full irqsave spinlocks.
10919          *
10920          * NB: An MMIO update of the plane base pointer will also
10921          * generate a page-flip completion irq, i.e. every modeset
10922          * is also accompanied by a spurious intel_prepare_page_flip().
10923          */
10924         spin_lock_irqsave(&dev->event_lock, flags);
10925         if (intel_crtc->unpin_work && page_flip_finished(intel_crtc))
10926                 atomic_inc_not_zero(&intel_crtc->unpin_work->pending);
10927         spin_unlock_irqrestore(&dev->event_lock, flags);
10928 }
10929
10930 static inline void intel_mark_page_flip_active(struct intel_unpin_work *work)
10931 {
10932         /* Ensure that the work item is consistent when activating it ... */
10933         smp_wmb();
10934         atomic_set(&work->pending, INTEL_FLIP_PENDING);
10935         /* and that it is marked active as soon as the irq could fire. */
10936         smp_wmb();
10937 }
10938
10939 static int intel_gen2_queue_flip(struct drm_device *dev,
10940                                  struct drm_crtc *crtc,
10941                                  struct drm_framebuffer *fb,
10942                                  struct drm_i915_gem_object *obj,
10943                                  struct drm_i915_gem_request *req,
10944                                  uint32_t flags)
10945 {
10946         struct intel_engine_cs *ring = req->ring;
10947         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10948         u32 flip_mask;
10949         int ret;
10950
10951         ret = intel_ring_begin(req, 6);
10952         if (ret)
10953                 return ret;
10954
10955         /* Can't queue multiple flips, so wait for the previous
10956          * one to finish before executing the next.
10957          */
10958         if (intel_crtc->plane)
10959                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
10960         else
10961                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
10962         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
10963         intel_ring_emit(ring, MI_NOOP);
10964         intel_ring_emit(ring, MI_DISPLAY_FLIP |
10965                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
10966         intel_ring_emit(ring, fb->pitches[0]);
10967         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
10968         intel_ring_emit(ring, 0); /* aux display base address, unused */
10969
10970         intel_mark_page_flip_active(intel_crtc->unpin_work);
10971         return 0;
10972 }
10973
10974 static int intel_gen3_queue_flip(struct drm_device *dev,
10975                                  struct drm_crtc *crtc,
10976                                  struct drm_framebuffer *fb,
10977                                  struct drm_i915_gem_object *obj,
10978                                  struct drm_i915_gem_request *req,
10979                                  uint32_t flags)
10980 {
10981         struct intel_engine_cs *ring = req->ring;
10982         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10983         u32 flip_mask;
10984         int ret;
10985
10986         ret = intel_ring_begin(req, 6);
10987         if (ret)
10988                 return ret;
10989
10990         if (intel_crtc->plane)
10991                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
10992         else
10993                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
10994         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
10995         intel_ring_emit(ring, MI_NOOP);
10996         intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 |
10997                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
10998         intel_ring_emit(ring, fb->pitches[0]);
10999         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
11000         intel_ring_emit(ring, MI_NOOP);
11001
11002         intel_mark_page_flip_active(intel_crtc->unpin_work);
11003         return 0;
11004 }
11005
11006 static int intel_gen4_queue_flip(struct drm_device *dev,
11007                                  struct drm_crtc *crtc,
11008                                  struct drm_framebuffer *fb,
11009                                  struct drm_i915_gem_object *obj,
11010                                  struct drm_i915_gem_request *req,
11011                                  uint32_t flags)
11012 {
11013         struct intel_engine_cs *ring = req->ring;
11014         struct drm_i915_private *dev_priv = dev->dev_private;
11015         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11016         uint32_t pf, pipesrc;
11017         int ret;
11018
11019         ret = intel_ring_begin(req, 4);
11020         if (ret)
11021                 return ret;
11022
11023         /* i965+ uses the linear or tiled offsets from the
11024          * Display Registers (which do not change across a page-flip)
11025          * so we need only reprogram the base address.
11026          */
11027         intel_ring_emit(ring, MI_DISPLAY_FLIP |
11028                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
11029         intel_ring_emit(ring, fb->pitches[0]);
11030         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset |
11031                         obj->tiling_mode);
11032
11033         /* XXX Enabling the panel-fitter across page-flip is so far
11034          * untested on non-native modes, so ignore it for now.
11035          * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
11036          */
11037         pf = 0;
11038         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
11039         intel_ring_emit(ring, pf | pipesrc);
11040
11041         intel_mark_page_flip_active(intel_crtc->unpin_work);
11042         return 0;
11043 }
11044
11045 static int intel_gen6_queue_flip(struct drm_device *dev,
11046                                  struct drm_crtc *crtc,
11047                                  struct drm_framebuffer *fb,
11048                                  struct drm_i915_gem_object *obj,
11049                                  struct drm_i915_gem_request *req,
11050                                  uint32_t flags)
11051 {
11052         struct intel_engine_cs *ring = req->ring;
11053         struct drm_i915_private *dev_priv = dev->dev_private;
11054         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11055         uint32_t pf, pipesrc;
11056         int ret;
11057
11058         ret = intel_ring_begin(req, 4);
11059         if (ret)
11060                 return ret;
11061
11062         intel_ring_emit(ring, MI_DISPLAY_FLIP |
11063                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
11064         intel_ring_emit(ring, fb->pitches[0] | obj->tiling_mode);
11065         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
11066
11067         /* Contrary to the suggestions in the documentation,
11068          * "Enable Panel Fitter" does not seem to be required when page
11069          * flipping with a non-native mode, and worse causes a normal
11070          * modeset to fail.
11071          * pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE;
11072          */
11073         pf = 0;
11074         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
11075         intel_ring_emit(ring, pf | pipesrc);
11076
11077         intel_mark_page_flip_active(intel_crtc->unpin_work);
11078         return 0;
11079 }
11080
11081 static int intel_gen7_queue_flip(struct drm_device *dev,
11082                                  struct drm_crtc *crtc,
11083                                  struct drm_framebuffer *fb,
11084                                  struct drm_i915_gem_object *obj,
11085                                  struct drm_i915_gem_request *req,
11086                                  uint32_t flags)
11087 {
11088         struct intel_engine_cs *ring = req->ring;
11089         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11090         uint32_t plane_bit = 0;
11091         int len, ret;
11092
11093         switch (intel_crtc->plane) {
11094         case PLANE_A:
11095                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_A;
11096                 break;
11097         case PLANE_B:
11098                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_B;
11099                 break;
11100         case PLANE_C:
11101                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_C;
11102                 break;
11103         default:
11104                 WARN_ONCE(1, "unknown plane in flip command\n");
11105                 return -ENODEV;
11106         }
11107
11108         len = 4;
11109         if (ring->id == RCS) {
11110                 len += 6;
11111                 /*
11112                  * On Gen 8, SRM is now taking an extra dword to accommodate
11113                  * 48bits addresses, and we need a NOOP for the batch size to
11114                  * stay even.
11115                  */
11116                 if (IS_GEN8(dev))
11117                         len += 2;
11118         }
11119
11120         /*
11121          * BSpec MI_DISPLAY_FLIP for IVB:
11122          * "The full packet must be contained within the same cache line."
11123          *
11124          * Currently the LRI+SRM+MI_DISPLAY_FLIP all fit within the same
11125          * cacheline, if we ever start emitting more commands before
11126          * the MI_DISPLAY_FLIP we may need to first emit everything else,
11127          * then do the cacheline alignment, and finally emit the
11128          * MI_DISPLAY_FLIP.
11129          */
11130         ret = intel_ring_cacheline_align(req);
11131         if (ret)
11132                 return ret;
11133
11134         ret = intel_ring_begin(req, len);
11135         if (ret)
11136                 return ret;
11137
11138         /* Unmask the flip-done completion message. Note that the bspec says that
11139          * we should do this for both the BCS and RCS, and that we must not unmask
11140          * more than one flip event at any time (or ensure that one flip message
11141          * can be sent by waiting for flip-done prior to queueing new flips).
11142          * Experimentation says that BCS works despite DERRMR masking all
11143          * flip-done completion events and that unmasking all planes at once
11144          * for the RCS also doesn't appear to drop events. Setting the DERRMR
11145          * to zero does lead to lockups within MI_DISPLAY_FLIP.
11146          */
11147         if (ring->id == RCS) {
11148                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
11149                 intel_ring_emit(ring, DERRMR);
11150                 intel_ring_emit(ring, ~(DERRMR_PIPEA_PRI_FLIP_DONE |
11151                                         DERRMR_PIPEB_PRI_FLIP_DONE |
11152                                         DERRMR_PIPEC_PRI_FLIP_DONE));
11153                 if (IS_GEN8(dev))
11154                         intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8 |
11155                                               MI_SRM_LRM_GLOBAL_GTT);
11156                 else
11157                         intel_ring_emit(ring, MI_STORE_REGISTER_MEM |
11158                                               MI_SRM_LRM_GLOBAL_GTT);
11159                 intel_ring_emit(ring, DERRMR);
11160                 intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
11161                 if (IS_GEN8(dev)) {
11162                         intel_ring_emit(ring, 0);
11163                         intel_ring_emit(ring, MI_NOOP);
11164                 }
11165         }
11166
11167         intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | plane_bit);
11168         intel_ring_emit(ring, (fb->pitches[0] | obj->tiling_mode));
11169         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
11170         intel_ring_emit(ring, (MI_NOOP));
11171
11172         intel_mark_page_flip_active(intel_crtc->unpin_work);
11173         return 0;
11174 }
11175
11176 static bool use_mmio_flip(struct intel_engine_cs *ring,
11177                           struct drm_i915_gem_object *obj)
11178 {
11179         /*
11180          * This is not being used for older platforms, because
11181          * non-availability of flip done interrupt forces us to use
11182          * CS flips. Older platforms derive flip done using some clever
11183          * tricks involving the flip_pending status bits and vblank irqs.
11184          * So using MMIO flips there would disrupt this mechanism.
11185          */
11186
11187         if (ring == NULL)
11188                 return true;
11189
11190         if (INTEL_INFO(ring->dev)->gen < 5)
11191                 return false;
11192
11193         if (i915.use_mmio_flip < 0)
11194                 return false;
11195         else if (i915.use_mmio_flip > 0)
11196                 return true;
11197         else if (i915.enable_execlists)
11198                 return true;
11199         else
11200                 return ring != i915_gem_request_get_ring(obj->last_write_req);
11201 }
11202
11203 static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
11204                              struct intel_unpin_work *work)
11205 {
11206         struct drm_device *dev = intel_crtc->base.dev;
11207         struct drm_i915_private *dev_priv = dev->dev_private;
11208         struct drm_framebuffer *fb = intel_crtc->base.primary->fb;
11209         const enum pipe pipe = intel_crtc->pipe;
11210         u32 ctl, stride;
11211
11212         ctl = I915_READ(PLANE_CTL(pipe, 0));
11213         ctl &= ~PLANE_CTL_TILED_MASK;
11214         switch (fb->modifier[0]) {
11215         case DRM_FORMAT_MOD_NONE:
11216                 break;
11217         case I915_FORMAT_MOD_X_TILED:
11218                 ctl |= PLANE_CTL_TILED_X;
11219                 break;
11220         case I915_FORMAT_MOD_Y_TILED:
11221                 ctl |= PLANE_CTL_TILED_Y;
11222                 break;
11223         case I915_FORMAT_MOD_Yf_TILED:
11224                 ctl |= PLANE_CTL_TILED_YF;
11225                 break;
11226         default:
11227                 MISSING_CASE(fb->modifier[0]);
11228         }
11229
11230         /*
11231          * The stride is either expressed as a multiple of 64 bytes chunks for
11232          * linear buffers or in number of tiles for tiled buffers.
11233          */
11234         stride = fb->pitches[0] /
11235                  intel_fb_stride_alignment(dev, fb->modifier[0],
11236                                            fb->pixel_format);
11237
11238         /*
11239          * Both PLANE_CTL and PLANE_STRIDE are not updated on vblank but on
11240          * PLANE_SURF updates, the update is then guaranteed to be atomic.
11241          */
11242         I915_WRITE(PLANE_CTL(pipe, 0), ctl);
11243         I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
11244
11245         I915_WRITE(PLANE_SURF(pipe, 0), work->gtt_offset);
11246         POSTING_READ(PLANE_SURF(pipe, 0));
11247 }
11248
11249 static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc,
11250                              struct intel_unpin_work *work)
11251 {
11252         struct drm_device *dev = intel_crtc->base.dev;
11253         struct drm_i915_private *dev_priv = dev->dev_private;
11254         struct intel_framebuffer *intel_fb =
11255                 to_intel_framebuffer(intel_crtc->base.primary->fb);
11256         struct drm_i915_gem_object *obj = intel_fb->obj;
11257         u32 dspcntr;
11258         u32 reg;
11259
11260         reg = DSPCNTR(intel_crtc->plane);
11261         dspcntr = I915_READ(reg);
11262
11263         if (obj->tiling_mode != I915_TILING_NONE)
11264                 dspcntr |= DISPPLANE_TILED;
11265         else
11266                 dspcntr &= ~DISPPLANE_TILED;
11267
11268         I915_WRITE(reg, dspcntr);
11269
11270         I915_WRITE(DSPSURF(intel_crtc->plane), work->gtt_offset);
11271         POSTING_READ(DSPSURF(intel_crtc->plane));
11272 }
11273
11274 /*
11275  * XXX: This is the temporary way to update the plane registers until we get
11276  * around to using the usual plane update functions for MMIO flips
11277  */
11278 static void intel_do_mmio_flip(struct intel_mmio_flip *mmio_flip)
11279 {
11280         struct intel_crtc *crtc = mmio_flip->crtc;
11281         struct intel_unpin_work *work;
11282
11283         spin_lock_irq(&crtc->base.dev->event_lock);
11284         work = crtc->unpin_work;
11285         spin_unlock_irq(&crtc->base.dev->event_lock);
11286         if (work == NULL)
11287                 return;
11288
11289         intel_mark_page_flip_active(work);
11290
11291         intel_pipe_update_start(crtc);
11292
11293         if (INTEL_INFO(mmio_flip->i915)->gen >= 9)
11294                 skl_do_mmio_flip(crtc, work);
11295         else
11296                 /* use_mmio_flip() retricts MMIO flips to ilk+ */
11297                 ilk_do_mmio_flip(crtc, work);
11298
11299         intel_pipe_update_end(crtc);
11300 }
11301
11302 static void intel_mmio_flip_work_func(struct work_struct *work)
11303 {
11304         struct intel_mmio_flip *mmio_flip =
11305                 container_of(work, struct intel_mmio_flip, work);
11306
11307         if (mmio_flip->req) {
11308                 WARN_ON(__i915_wait_request(mmio_flip->req,
11309                                             mmio_flip->crtc->reset_counter,
11310                                             false, NULL,
11311                                             &mmio_flip->i915->rps.mmioflips));
11312                 i915_gem_request_unreference__unlocked(mmio_flip->req);
11313         }
11314
11315         intel_do_mmio_flip(mmio_flip);
11316         kfree(mmio_flip);
11317 }
11318
11319 static int intel_queue_mmio_flip(struct drm_device *dev,
11320                                  struct drm_crtc *crtc,
11321                                  struct drm_framebuffer *fb,
11322                                  struct drm_i915_gem_object *obj,
11323                                  struct intel_engine_cs *ring,
11324                                  uint32_t flags)
11325 {
11326         struct intel_mmio_flip *mmio_flip;
11327
11328         mmio_flip = kmalloc(sizeof(*mmio_flip), GFP_KERNEL);
11329         if (mmio_flip == NULL)
11330                 return -ENOMEM;
11331
11332         mmio_flip->i915 = to_i915(dev);
11333         mmio_flip->req = i915_gem_request_reference(obj->last_write_req);
11334         mmio_flip->crtc = to_intel_crtc(crtc);
11335
11336         INIT_WORK(&mmio_flip->work, intel_mmio_flip_work_func);
11337         schedule_work(&mmio_flip->work);
11338
11339         return 0;
11340 }
11341
11342 static int intel_default_queue_flip(struct drm_device *dev,
11343                                     struct drm_crtc *crtc,
11344                                     struct drm_framebuffer *fb,
11345                                     struct drm_i915_gem_object *obj,
11346                                     struct drm_i915_gem_request *req,
11347                                     uint32_t flags)
11348 {
11349         return -ENODEV;
11350 }
11351
11352 static bool __intel_pageflip_stall_check(struct drm_device *dev,
11353                                          struct drm_crtc *crtc)
11354 {
11355         struct drm_i915_private *dev_priv = dev->dev_private;
11356         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11357         struct intel_unpin_work *work = intel_crtc->unpin_work;
11358         u32 addr;
11359
11360         if (atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE)
11361                 return true;
11362
11363         if (atomic_read(&work->pending) < INTEL_FLIP_PENDING)
11364                 return false;
11365
11366         if (!work->enable_stall_check)
11367                 return false;
11368
11369         if (work->flip_ready_vblank == 0) {
11370                 if (work->flip_queued_req &&
11371                     !i915_gem_request_completed(work->flip_queued_req, true))
11372                         return false;
11373
11374                 work->flip_ready_vblank = drm_crtc_vblank_count(crtc);
11375         }
11376
11377         if (drm_crtc_vblank_count(crtc) - work->flip_ready_vblank < 3)
11378                 return false;
11379
11380         /* Potential stall - if we see that the flip has happened,
11381          * assume a missed interrupt. */
11382         if (INTEL_INFO(dev)->gen >= 4)
11383                 addr = I915_HI_DISPBASE(I915_READ(DSPSURF(intel_crtc->plane)));
11384         else
11385                 addr = I915_READ(DSPADDR(intel_crtc->plane));
11386
11387         /* There is a potential issue here with a false positive after a flip
11388          * to the same address. We could address this by checking for a
11389          * non-incrementing frame counter.
11390          */
11391         return addr == work->gtt_offset;
11392 }
11393
11394 void intel_check_page_flip(struct drm_device *dev, int pipe)
11395 {
11396         struct drm_i915_private *dev_priv = dev->dev_private;
11397         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
11398         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11399         struct intel_unpin_work *work;
11400
11401         WARN_ON(!in_interrupt());
11402
11403         if (crtc == NULL)
11404                 return;
11405
11406         spin_lock(&dev->event_lock);
11407         work = intel_crtc->unpin_work;
11408         if (work != NULL && __intel_pageflip_stall_check(dev, crtc)) {
11409                 WARN_ONCE(1, "Kicking stuck page flip: queued at %d, now %d\n",
11410                          work->flip_queued_vblank, drm_vblank_count(dev, pipe));
11411                 page_flip_completed(intel_crtc);
11412                 work = NULL;
11413         }
11414         if (work != NULL &&
11415             drm_vblank_count(dev, pipe) - work->flip_queued_vblank > 1)
11416                 intel_queue_rps_boost_for_request(dev, work->flip_queued_req);
11417         spin_unlock(&dev->event_lock);
11418 }
11419
11420 static int intel_crtc_page_flip(struct drm_crtc *crtc,
11421                                 struct drm_framebuffer *fb,
11422                                 struct drm_pending_vblank_event *event,
11423                                 uint32_t page_flip_flags)
11424 {
11425         struct drm_device *dev = crtc->dev;
11426         struct drm_i915_private *dev_priv = dev->dev_private;
11427         struct drm_framebuffer *old_fb = crtc->primary->fb;
11428         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
11429         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11430         struct drm_plane *primary = crtc->primary;
11431         enum pipe pipe = intel_crtc->pipe;
11432         struct intel_unpin_work *work;
11433         struct intel_engine_cs *ring;
11434         bool mmio_flip;
11435         struct drm_i915_gem_request *request = NULL;
11436         int ret;
11437
11438         /*
11439          * drm_mode_page_flip_ioctl() should already catch this, but double
11440          * check to be safe.  In the future we may enable pageflipping from
11441          * a disabled primary plane.
11442          */
11443         if (WARN_ON(intel_fb_obj(old_fb) == NULL))
11444                 return -EBUSY;
11445
11446         /* Can't change pixel format via MI display flips. */
11447         if (fb->pixel_format != crtc->primary->fb->pixel_format)
11448                 return -EINVAL;
11449
11450         /*
11451          * TILEOFF/LINOFF registers can't be changed via MI display flips.
11452          * Note that pitch changes could also affect these register.
11453          */
11454         if (INTEL_INFO(dev)->gen > 3 &&
11455             (fb->offsets[0] != crtc->primary->fb->offsets[0] ||
11456              fb->pitches[0] != crtc->primary->fb->pitches[0]))
11457                 return -EINVAL;
11458
11459         if (i915_terminally_wedged(&dev_priv->gpu_error))
11460                 goto out_hang;
11461
11462         work = kzalloc(sizeof(*work), GFP_KERNEL);
11463         if (work == NULL)
11464                 return -ENOMEM;
11465
11466         work->event = event;
11467         work->crtc = crtc;
11468         work->old_fb = old_fb;
11469         INIT_WORK(&work->work, intel_unpin_work_fn);
11470
11471         ret = drm_crtc_vblank_get(crtc);
11472         if (ret)
11473                 goto free_work;
11474
11475         /* We borrow the event spin lock for protecting unpin_work */
11476         spin_lock_irq(&dev->event_lock);
11477         if (intel_crtc->unpin_work) {
11478                 /* Before declaring the flip queue wedged, check if
11479                  * the hardware completed the operation behind our backs.
11480                  */
11481                 if (__intel_pageflip_stall_check(dev, crtc)) {
11482                         DRM_DEBUG_DRIVER("flip queue: previous flip completed, continuing\n");
11483                         page_flip_completed(intel_crtc);
11484                 } else {
11485                         DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
11486                         spin_unlock_irq(&dev->event_lock);
11487
11488                         drm_crtc_vblank_put(crtc);
11489                         kfree(work);
11490                         return -EBUSY;
11491                 }
11492         }
11493         intel_crtc->unpin_work = work;
11494         spin_unlock_irq(&dev->event_lock);
11495
11496         if (atomic_read(&intel_crtc->unpin_work_count) >= 2)
11497                 flush_workqueue(dev_priv->wq);
11498
11499         /* Reference the objects for the scheduled work. */
11500         drm_framebuffer_reference(work->old_fb);
11501         drm_gem_object_reference(&obj->base);
11502
11503         crtc->primary->fb = fb;
11504         update_state_fb(crtc->primary);
11505
11506         work->pending_flip_obj = obj;
11507
11508         ret = i915_mutex_lock_interruptible(dev);
11509         if (ret)
11510                 goto cleanup;
11511
11512         atomic_inc(&intel_crtc->unpin_work_count);
11513         intel_crtc->reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
11514
11515         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
11516                 work->flip_count = I915_READ(PIPE_FLIPCOUNT_G4X(pipe)) + 1;
11517
11518         if (IS_VALLEYVIEW(dev)) {
11519                 ring = &dev_priv->ring[BCS];
11520                 if (obj->tiling_mode != intel_fb_obj(work->old_fb)->tiling_mode)
11521                         /* vlv: DISPLAY_FLIP fails to change tiling */
11522                         ring = NULL;
11523         } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
11524                 ring = &dev_priv->ring[BCS];
11525         } else if (INTEL_INFO(dev)->gen >= 7) {
11526                 ring = i915_gem_request_get_ring(obj->last_write_req);
11527                 if (ring == NULL || ring->id != RCS)
11528                         ring = &dev_priv->ring[BCS];
11529         } else {
11530                 ring = &dev_priv->ring[RCS];
11531         }
11532
11533         mmio_flip = use_mmio_flip(ring, obj);
11534
11535         /* When using CS flips, we want to emit semaphores between rings.
11536          * However, when using mmio flips we will create a task to do the
11537          * synchronisation, so all we want here is to pin the framebuffer
11538          * into the display plane and skip any waits.
11539          */
11540         ret = intel_pin_and_fence_fb_obj(crtc->primary, fb,
11541                                          crtc->primary->state,
11542                                          mmio_flip ? i915_gem_request_get_ring(obj->last_write_req) : ring, &request);
11543         if (ret)
11544                 goto cleanup_pending;
11545
11546         work->gtt_offset = intel_plane_obj_offset(to_intel_plane(primary),
11547                                                   obj, 0);
11548         work->gtt_offset += intel_crtc->dspaddr_offset;
11549
11550         if (mmio_flip) {
11551                 ret = intel_queue_mmio_flip(dev, crtc, fb, obj, ring,
11552                                             page_flip_flags);
11553                 if (ret)
11554                         goto cleanup_unpin;
11555
11556                 i915_gem_request_assign(&work->flip_queued_req,
11557                                         obj->last_write_req);
11558         } else {
11559                 if (!request) {
11560                         ret = i915_gem_request_alloc(ring, ring->default_context, &request);
11561                         if (ret)
11562                                 goto cleanup_unpin;
11563                 }
11564
11565                 ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, request,
11566                                                    page_flip_flags);
11567                 if (ret)
11568                         goto cleanup_unpin;
11569
11570                 i915_gem_request_assign(&work->flip_queued_req, request);
11571         }
11572
11573         if (request)
11574                 i915_add_request_no_flush(request);
11575
11576         work->flip_queued_vblank = drm_crtc_vblank_count(crtc);
11577         work->enable_stall_check = true;
11578
11579         i915_gem_track_fb(intel_fb_obj(work->old_fb), obj,
11580                           to_intel_plane(primary)->frontbuffer_bit);
11581         mutex_unlock(&dev->struct_mutex);
11582
11583         intel_fbc_disable_crtc(intel_crtc);
11584         intel_frontbuffer_flip_prepare(dev,
11585                                        to_intel_plane(primary)->frontbuffer_bit);
11586
11587         trace_i915_flip_request(intel_crtc->plane, obj);
11588
11589         return 0;
11590
11591 cleanup_unpin:
11592         intel_unpin_fb_obj(fb, crtc->primary->state);
11593 cleanup_pending:
11594         if (request)
11595                 i915_gem_request_cancel(request);
11596         atomic_dec(&intel_crtc->unpin_work_count);
11597         mutex_unlock(&dev->struct_mutex);
11598 cleanup:
11599         crtc->primary->fb = old_fb;
11600         update_state_fb(crtc->primary);
11601
11602         drm_gem_object_unreference_unlocked(&obj->base);
11603         drm_framebuffer_unreference(work->old_fb);
11604
11605         spin_lock_irq(&dev->event_lock);
11606         intel_crtc->unpin_work = NULL;
11607         spin_unlock_irq(&dev->event_lock);
11608
11609         drm_crtc_vblank_put(crtc);
11610 free_work:
11611         kfree(work);
11612
11613         if (ret == -EIO) {
11614                 struct drm_atomic_state *state;
11615                 struct drm_plane_state *plane_state;
11616
11617 out_hang:
11618                 state = drm_atomic_state_alloc(dev);
11619                 if (!state)
11620                         return -ENOMEM;
11621                 state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc);
11622
11623 retry:
11624                 plane_state = drm_atomic_get_plane_state(state, primary);
11625                 ret = PTR_ERR_OR_ZERO(plane_state);
11626                 if (!ret) {
11627                         drm_atomic_set_fb_for_plane(plane_state, fb);
11628
11629                         ret = drm_atomic_set_crtc_for_plane(plane_state, crtc);
11630                         if (!ret)
11631                                 ret = drm_atomic_commit(state);
11632                 }
11633
11634                 if (ret == -EDEADLK) {
11635                         drm_modeset_backoff(state->acquire_ctx);
11636                         drm_atomic_state_clear(state);
11637                         goto retry;
11638                 }
11639
11640                 if (ret)
11641                         drm_atomic_state_free(state);
11642
11643                 if (ret == 0 && event) {
11644                         spin_lock_irq(&dev->event_lock);
11645                         drm_send_vblank_event(dev, pipe, event);
11646                         spin_unlock_irq(&dev->event_lock);
11647                 }
11648         }
11649         return ret;
11650 }
11651
11652
11653 /**
11654  * intel_wm_need_update - Check whether watermarks need updating
11655  * @plane: drm plane
11656  * @state: new plane state
11657  *
11658  * Check current plane state versus the new one to determine whether
11659  * watermarks need to be recalculated.
11660  *
11661  * Returns true or false.
11662  */
11663 static bool intel_wm_need_update(struct drm_plane *plane,
11664                                  struct drm_plane_state *state)
11665 {
11666         /* Update watermarks on tiling changes. */
11667         if (!plane->state->fb || !state->fb ||
11668             plane->state->fb->modifier[0] != state->fb->modifier[0] ||
11669             plane->state->rotation != state->rotation)
11670                 return true;
11671
11672         if (plane->state->crtc_w != state->crtc_w)
11673                 return true;
11674
11675         return false;
11676 }
11677
11678 int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
11679                                     struct drm_plane_state *plane_state)
11680 {
11681         struct drm_crtc *crtc = crtc_state->crtc;
11682         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11683         struct drm_plane *plane = plane_state->plane;
11684         struct drm_device *dev = crtc->dev;
11685         struct drm_i915_private *dev_priv = dev->dev_private;
11686         struct intel_plane_state *old_plane_state =
11687                 to_intel_plane_state(plane->state);
11688         int idx = intel_crtc->base.base.id, ret;
11689         int i = drm_plane_index(plane);
11690         bool mode_changed = needs_modeset(crtc_state);
11691         bool was_crtc_enabled = crtc->state->active;
11692         bool is_crtc_enabled = crtc_state->active;
11693
11694         bool turn_off, turn_on, visible, was_visible;
11695         struct drm_framebuffer *fb = plane_state->fb;
11696
11697         if (crtc_state && INTEL_INFO(dev)->gen >= 9 &&
11698             plane->type != DRM_PLANE_TYPE_CURSOR) {
11699                 ret = skl_update_scaler_plane(
11700                         to_intel_crtc_state(crtc_state),
11701                         to_intel_plane_state(plane_state));
11702                 if (ret)
11703                         return ret;
11704         }
11705
11706         /*
11707          * Disabling a plane is always okay; we just need to update
11708          * fb tracking in a special way since cleanup_fb() won't
11709          * get called by the plane helpers.
11710          */
11711         if (old_plane_state->base.fb && !fb)
11712                 intel_crtc->atomic.disabled_planes |= 1 << i;
11713
11714         was_visible = old_plane_state->visible;
11715         visible = to_intel_plane_state(plane_state)->visible;
11716
11717         if (!was_crtc_enabled && WARN_ON(was_visible))
11718                 was_visible = false;
11719
11720         if (!is_crtc_enabled && WARN_ON(visible))
11721                 visible = false;
11722
11723         if (!was_visible && !visible)
11724                 return 0;
11725
11726         turn_off = was_visible && (!visible || mode_changed);
11727         turn_on = visible && (!was_visible || mode_changed);
11728
11729         DRM_DEBUG_ATOMIC("[CRTC:%i] has [PLANE:%i] with fb %i\n", idx,
11730                          plane->base.id, fb ? fb->base.id : -1);
11731
11732         DRM_DEBUG_ATOMIC("[PLANE:%i] visible %i -> %i, off %i, on %i, ms %i\n",
11733                          plane->base.id, was_visible, visible,
11734                          turn_off, turn_on, mode_changed);
11735
11736         if (turn_on) {
11737                 intel_crtc->atomic.update_wm_pre = true;
11738                 /* must disable cxsr around plane enable/disable */
11739                 if (plane->type != DRM_PLANE_TYPE_CURSOR) {
11740                         intel_crtc->atomic.disable_cxsr = true;
11741                         /* to potentially re-enable cxsr */
11742                         intel_crtc->atomic.wait_vblank = true;
11743                         intel_crtc->atomic.update_wm_post = true;
11744                 }
11745         } else if (turn_off) {
11746                 intel_crtc->atomic.update_wm_post = true;
11747                 /* must disable cxsr around plane enable/disable */
11748                 if (plane->type != DRM_PLANE_TYPE_CURSOR) {
11749                         if (is_crtc_enabled)
11750                                 intel_crtc->atomic.wait_vblank = true;
11751                         intel_crtc->atomic.disable_cxsr = true;
11752                 }
11753         } else if (intel_wm_need_update(plane, plane_state)) {
11754                 intel_crtc->atomic.update_wm_pre = true;
11755         }
11756
11757         if (visible || was_visible)
11758                 intel_crtc->atomic.fb_bits |=
11759                         to_intel_plane(plane)->frontbuffer_bit;
11760
11761         switch (plane->type) {
11762         case DRM_PLANE_TYPE_PRIMARY:
11763                 intel_crtc->atomic.wait_for_flips = true;
11764                 intel_crtc->atomic.pre_disable_primary = turn_off;
11765                 intel_crtc->atomic.post_enable_primary = turn_on;
11766
11767                 if (turn_off) {
11768                         /*
11769                          * FIXME: Actually if we will still have any other
11770                          * plane enabled on the pipe we could let IPS enabled
11771                          * still, but for now lets consider that when we make
11772                          * primary invisible by setting DSPCNTR to 0 on
11773                          * update_primary_plane function IPS needs to be
11774                          * disable.
11775                          */
11776                         intel_crtc->atomic.disable_ips = true;
11777
11778                         intel_crtc->atomic.disable_fbc = true;
11779                 }
11780
11781                 /*
11782                  * FBC does not work on some platforms for rotated
11783                  * planes, so disable it when rotation is not 0 and
11784                  * update it when rotation is set back to 0.
11785                  *
11786                  * FIXME: This is redundant with the fbc update done in
11787                  * the primary plane enable function except that that
11788                  * one is done too late. We eventually need to unify
11789                  * this.
11790                  */
11791
11792                 if (visible &&
11793                     INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) &&
11794                     dev_priv->fbc.crtc == intel_crtc &&
11795                     plane_state->rotation != BIT(DRM_ROTATE_0))
11796                         intel_crtc->atomic.disable_fbc = true;
11797
11798                 /*
11799                  * BDW signals flip done immediately if the plane
11800                  * is disabled, even if the plane enable is already
11801                  * armed to occur at the next vblank :(
11802                  */
11803                 if (turn_on && IS_BROADWELL(dev))
11804                         intel_crtc->atomic.wait_vblank = true;
11805
11806                 intel_crtc->atomic.update_fbc |= visible || mode_changed;
11807                 break;
11808         case DRM_PLANE_TYPE_CURSOR:
11809                 break;
11810         case DRM_PLANE_TYPE_OVERLAY:
11811                 if (turn_off && !mode_changed) {
11812                         intel_crtc->atomic.wait_vblank = true;
11813                         intel_crtc->atomic.update_sprite_watermarks |=
11814                                 1 << i;
11815                 }
11816         }
11817         return 0;
11818 }
11819
11820 static bool encoders_cloneable(const struct intel_encoder *a,
11821                                const struct intel_encoder *b)
11822 {
11823         /* masks could be asymmetric, so check both ways */
11824         return a == b || (a->cloneable & (1 << b->type) &&
11825                           b->cloneable & (1 << a->type));
11826 }
11827
11828 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
11829                                          struct intel_crtc *crtc,
11830                                          struct intel_encoder *encoder)
11831 {
11832         struct intel_encoder *source_encoder;
11833         struct drm_connector *connector;
11834         struct drm_connector_state *connector_state;
11835         int i;
11836
11837         for_each_connector_in_state(state, connector, connector_state, i) {
11838                 if (connector_state->crtc != &crtc->base)
11839                         continue;
11840
11841                 source_encoder =
11842                         to_intel_encoder(connector_state->best_encoder);
11843                 if (!encoders_cloneable(encoder, source_encoder))
11844                         return false;
11845         }
11846
11847         return true;
11848 }
11849
11850 static bool check_encoder_cloning(struct drm_atomic_state *state,
11851                                   struct intel_crtc *crtc)
11852 {
11853         struct intel_encoder *encoder;
11854         struct drm_connector *connector;
11855         struct drm_connector_state *connector_state;
11856         int i;
11857
11858         for_each_connector_in_state(state, connector, connector_state, i) {
11859                 if (connector_state->crtc != &crtc->base)
11860                         continue;
11861
11862                 encoder = to_intel_encoder(connector_state->best_encoder);
11863                 if (!check_single_encoder_cloning(state, crtc, encoder))
11864                         return false;
11865         }
11866
11867         return true;
11868 }
11869
11870 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
11871                                    struct drm_crtc_state *crtc_state)
11872 {
11873         struct drm_device *dev = crtc->dev;
11874         struct drm_i915_private *dev_priv = dev->dev_private;
11875         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11876         struct intel_crtc_state *pipe_config =
11877                 to_intel_crtc_state(crtc_state);
11878         struct drm_atomic_state *state = crtc_state->state;
11879         int ret;
11880         bool mode_changed = needs_modeset(crtc_state);
11881
11882         if (mode_changed && !check_encoder_cloning(state, intel_crtc)) {
11883                 DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
11884                 return -EINVAL;
11885         }
11886
11887         if (mode_changed && !crtc_state->active)
11888                 intel_crtc->atomic.update_wm_post = true;
11889
11890         if (mode_changed && crtc_state->enable &&
11891             dev_priv->display.crtc_compute_clock &&
11892             !WARN_ON(pipe_config->shared_dpll != DPLL_ID_PRIVATE)) {
11893                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
11894                                                            pipe_config);
11895                 if (ret)
11896                         return ret;
11897         }
11898
11899         ret = 0;
11900         if (INTEL_INFO(dev)->gen >= 9) {
11901                 if (mode_changed)
11902                         ret = skl_update_scaler_crtc(pipe_config);
11903
11904                 if (!ret)
11905                         ret = intel_atomic_setup_scalers(dev, intel_crtc,
11906                                                          pipe_config);
11907         }
11908
11909         return ret;
11910 }
11911
11912 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
11913         .mode_set_base_atomic = intel_pipe_set_base_atomic,
11914         .load_lut = intel_crtc_load_lut,
11915         .atomic_begin = intel_begin_crtc_commit,
11916         .atomic_flush = intel_finish_crtc_commit,
11917         .atomic_check = intel_crtc_atomic_check,
11918 };
11919
11920 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
11921 {
11922         struct intel_connector *connector;
11923
11924         for_each_intel_connector(dev, connector) {
11925                 if (connector->base.encoder) {
11926                         connector->base.state->best_encoder =
11927                                 connector->base.encoder;
11928                         connector->base.state->crtc =
11929                                 connector->base.encoder->crtc;
11930                 } else {
11931                         connector->base.state->best_encoder = NULL;
11932                         connector->base.state->crtc = NULL;
11933                 }
11934         }
11935 }
11936
11937 static void
11938 connected_sink_compute_bpp(struct intel_connector *connector,
11939                            struct intel_crtc_state *pipe_config)
11940 {
11941         int bpp = pipe_config->pipe_bpp;
11942
11943         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] checking for sink bpp constrains\n",
11944                 connector->base.base.id,
11945                 connector->base.name);
11946
11947         /* Don't use an invalid EDID bpc value */
11948         if (connector->base.display_info.bpc &&
11949             connector->base.display_info.bpc * 3 < bpp) {
11950                 DRM_DEBUG_KMS("clamping display bpp (was %d) to EDID reported max of %d\n",
11951                               bpp, connector->base.display_info.bpc*3);
11952                 pipe_config->pipe_bpp = connector->base.display_info.bpc*3;
11953         }
11954
11955         /* Clamp bpp to default limit on screens without EDID 1.4 */
11956         if (connector->base.display_info.bpc == 0) {
11957                 int type = connector->base.connector_type;
11958                 int clamp_bpp = 24;
11959
11960                 /* Fall back to 18 bpp when DP sink capability is unknown. */
11961                 if (type == DRM_MODE_CONNECTOR_DisplayPort ||
11962                     type == DRM_MODE_CONNECTOR_eDP)
11963                         clamp_bpp = 18;
11964
11965                 if (bpp > clamp_bpp) {
11966                         DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of %d\n",
11967                                       bpp, clamp_bpp);
11968                         pipe_config->pipe_bpp = clamp_bpp;
11969                 }
11970         }
11971 }
11972
11973 static int
11974 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
11975                           struct intel_crtc_state *pipe_config)
11976 {
11977         struct drm_device *dev = crtc->base.dev;
11978         struct drm_atomic_state *state;
11979         struct drm_connector *connector;
11980         struct drm_connector_state *connector_state;
11981         int bpp, i;
11982
11983         if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)))
11984                 bpp = 10*3;
11985         else if (INTEL_INFO(dev)->gen >= 5)
11986                 bpp = 12*3;
11987         else
11988                 bpp = 8*3;
11989
11990
11991         pipe_config->pipe_bpp = bpp;
11992
11993         state = pipe_config->base.state;
11994
11995         /* Clamp display bpp to EDID value */
11996         for_each_connector_in_state(state, connector, connector_state, i) {
11997                 if (connector_state->crtc != &crtc->base)
11998                         continue;
11999
12000                 connected_sink_compute_bpp(to_intel_connector(connector),
12001                                            pipe_config);
12002         }
12003
12004         return bpp;
12005 }
12006
12007 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
12008 {
12009         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
12010                         "type: 0x%x flags: 0x%x\n",
12011                 mode->crtc_clock,
12012                 mode->crtc_hdisplay, mode->crtc_hsync_start,
12013                 mode->crtc_hsync_end, mode->crtc_htotal,
12014                 mode->crtc_vdisplay, mode->crtc_vsync_start,
12015                 mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags);
12016 }
12017
12018 static void intel_dump_pipe_config(struct intel_crtc *crtc,
12019                                    struct intel_crtc_state *pipe_config,
12020                                    const char *context)
12021 {
12022         struct drm_device *dev = crtc->base.dev;
12023         struct drm_plane *plane;
12024         struct intel_plane *intel_plane;
12025         struct intel_plane_state *state;
12026         struct drm_framebuffer *fb;
12027
12028         DRM_DEBUG_KMS("[CRTC:%d]%s config %p for pipe %c\n", crtc->base.base.id,
12029                       context, pipe_config, pipe_name(crtc->pipe));
12030
12031         DRM_DEBUG_KMS("cpu_transcoder: %c\n", transcoder_name(pipe_config->cpu_transcoder));
12032         DRM_DEBUG_KMS("pipe bpp: %i, dithering: %i\n",
12033                       pipe_config->pipe_bpp, pipe_config->dither);
12034         DRM_DEBUG_KMS("fdi/pch: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12035                       pipe_config->has_pch_encoder,
12036                       pipe_config->fdi_lanes,
12037                       pipe_config->fdi_m_n.gmch_m, pipe_config->fdi_m_n.gmch_n,
12038                       pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n,
12039                       pipe_config->fdi_m_n.tu);
12040         DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12041                       pipe_config->has_dp_encoder,
12042                       pipe_config->lane_count,
12043                       pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n,
12044                       pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n,
12045                       pipe_config->dp_m_n.tu);
12046
12047         DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n",
12048                       pipe_config->has_dp_encoder,
12049                       pipe_config->lane_count,
12050                       pipe_config->dp_m2_n2.gmch_m,
12051                       pipe_config->dp_m2_n2.gmch_n,
12052                       pipe_config->dp_m2_n2.link_m,
12053                       pipe_config->dp_m2_n2.link_n,
12054                       pipe_config->dp_m2_n2.tu);
12055
12056         DRM_DEBUG_KMS("audio: %i, infoframes: %i\n",
12057                       pipe_config->has_audio,
12058                       pipe_config->has_infoframe);
12059
12060         DRM_DEBUG_KMS("requested mode:\n");
12061         drm_mode_debug_printmodeline(&pipe_config->base.mode);
12062         DRM_DEBUG_KMS("adjusted mode:\n");
12063         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
12064         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
12065         DRM_DEBUG_KMS("port clock: %d\n", pipe_config->port_clock);
12066         DRM_DEBUG_KMS("pipe src size: %dx%d\n",
12067                       pipe_config->pipe_src_w, pipe_config->pipe_src_h);
12068         DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12069                       crtc->num_scalers,
12070                       pipe_config->scaler_state.scaler_users,
12071                       pipe_config->scaler_state.scaler_id);
12072         DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12073                       pipe_config->gmch_pfit.control,
12074                       pipe_config->gmch_pfit.pgm_ratios,
12075                       pipe_config->gmch_pfit.lvds_border_bits);
12076         DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n",
12077                       pipe_config->pch_pfit.pos,
12078                       pipe_config->pch_pfit.size,
12079                       pipe_config->pch_pfit.enabled ? "enabled" : "disabled");
12080         DRM_DEBUG_KMS("ips: %i\n", pipe_config->ips_enabled);
12081         DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide);
12082
12083         if (IS_BROXTON(dev)) {
12084                 DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x,"
12085                               "pll0: 0x%x, pll1: 0x%x, pll2: 0x%x, pll3: 0x%x, "
12086                               "pll6: 0x%x, pll8: 0x%x, pll9: 0x%x, pll10: 0x%x, pcsdw12: 0x%x\n",
12087                               pipe_config->ddi_pll_sel,
12088                               pipe_config->dpll_hw_state.ebb0,
12089                               pipe_config->dpll_hw_state.ebb4,
12090                               pipe_config->dpll_hw_state.pll0,
12091                               pipe_config->dpll_hw_state.pll1,
12092                               pipe_config->dpll_hw_state.pll2,
12093                               pipe_config->dpll_hw_state.pll3,
12094                               pipe_config->dpll_hw_state.pll6,
12095                               pipe_config->dpll_hw_state.pll8,
12096                               pipe_config->dpll_hw_state.pll9,
12097                               pipe_config->dpll_hw_state.pll10,
12098                               pipe_config->dpll_hw_state.pcsdw12);
12099         } else if (IS_SKYLAKE(dev)) {
12100                 DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: "
12101                               "ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n",
12102                               pipe_config->ddi_pll_sel,
12103                               pipe_config->dpll_hw_state.ctrl1,
12104                               pipe_config->dpll_hw_state.cfgcr1,
12105                               pipe_config->dpll_hw_state.cfgcr2);
12106         } else if (HAS_DDI(dev)) {
12107                 DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: wrpll: 0x%x spll: 0x%x\n",
12108                               pipe_config->ddi_pll_sel,
12109                               pipe_config->dpll_hw_state.wrpll,
12110                               pipe_config->dpll_hw_state.spll);
12111         } else {
12112                 DRM_DEBUG_KMS("dpll_hw_state: dpll: 0x%x, dpll_md: 0x%x, "
12113                               "fp0: 0x%x, fp1: 0x%x\n",
12114                               pipe_config->dpll_hw_state.dpll,
12115                               pipe_config->dpll_hw_state.dpll_md,
12116                               pipe_config->dpll_hw_state.fp0,
12117                               pipe_config->dpll_hw_state.fp1);
12118         }
12119
12120         DRM_DEBUG_KMS("planes on this crtc\n");
12121         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
12122                 intel_plane = to_intel_plane(plane);
12123                 if (intel_plane->pipe != crtc->pipe)
12124                         continue;
12125
12126                 state = to_intel_plane_state(plane->state);
12127                 fb = state->base.fb;
12128                 if (!fb) {
12129                         DRM_DEBUG_KMS("%s PLANE:%d plane: %u.%u idx: %d "
12130                                 "disabled, scaler_id = %d\n",
12131                                 plane->type == DRM_PLANE_TYPE_CURSOR ? "CURSOR" : "STANDARD",
12132                                 plane->base.id, intel_plane->pipe,
12133                                 (crtc->base.primary == plane) ? 0 : intel_plane->plane + 1,
12134                                 drm_plane_index(plane), state->scaler_id);
12135                         continue;
12136                 }
12137
12138                 DRM_DEBUG_KMS("%s PLANE:%d plane: %u.%u idx: %d enabled",
12139                         plane->type == DRM_PLANE_TYPE_CURSOR ? "CURSOR" : "STANDARD",
12140                         plane->base.id, intel_plane->pipe,
12141                         crtc->base.primary == plane ? 0 : intel_plane->plane + 1,
12142                         drm_plane_index(plane));
12143                 DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = 0x%x",
12144                         fb->base.id, fb->width, fb->height, fb->pixel_format);
12145                 DRM_DEBUG_KMS("\tscaler:%d src (%u, %u) %ux%u dst (%u, %u) %ux%u\n",
12146                         state->scaler_id,
12147                         state->src.x1 >> 16, state->src.y1 >> 16,
12148                         drm_rect_width(&state->src) >> 16,
12149                         drm_rect_height(&state->src) >> 16,
12150                         state->dst.x1, state->dst.y1,
12151                         drm_rect_width(&state->dst), drm_rect_height(&state->dst));
12152         }
12153 }
12154
12155 static bool check_digital_port_conflicts(struct drm_atomic_state *state)
12156 {
12157         struct drm_device *dev = state->dev;
12158         struct drm_connector *connector;
12159         unsigned int used_ports = 0;
12160
12161         /*
12162          * Walk the connector list instead of the encoder
12163          * list to detect the problem on ddi platforms
12164          * where there's just one encoder per digital port.
12165          */
12166         drm_for_each_connector(connector, dev) {
12167                 struct drm_connector_state *connector_state;
12168                 struct intel_encoder *encoder;
12169
12170                 connector_state = drm_atomic_get_existing_connector_state(state, connector);
12171                 if (!connector_state)
12172                         connector_state = connector->state;
12173
12174                 if (!connector_state->best_encoder)
12175                         continue;
12176
12177                 encoder = to_intel_encoder(connector_state->best_encoder);
12178
12179                 WARN_ON(!connector_state->crtc);
12180
12181                 switch (encoder->type) {
12182                         unsigned int port_mask;
12183                 case INTEL_OUTPUT_UNKNOWN:
12184                         if (WARN_ON(!HAS_DDI(dev)))
12185                                 break;
12186                 case INTEL_OUTPUT_DISPLAYPORT:
12187                 case INTEL_OUTPUT_HDMI:
12188                 case INTEL_OUTPUT_EDP:
12189                         port_mask = 1 << enc_to_dig_port(&encoder->base)->port;
12190
12191                         /* the same port mustn't appear more than once */
12192                         if (used_ports & port_mask)
12193                                 return false;
12194
12195                         used_ports |= port_mask;
12196                 default:
12197                         break;
12198                 }
12199         }
12200
12201         return true;
12202 }
12203
12204 static void
12205 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
12206 {
12207         struct drm_crtc_state tmp_state;
12208         struct intel_crtc_scaler_state scaler_state;
12209         struct intel_dpll_hw_state dpll_hw_state;
12210         enum intel_dpll_id shared_dpll;
12211         uint32_t ddi_pll_sel;
12212         bool force_thru;
12213
12214         /* FIXME: before the switch to atomic started, a new pipe_config was
12215          * kzalloc'd. Code that depends on any field being zero should be
12216          * fixed, so that the crtc_state can be safely duplicated. For now,
12217          * only fields that are know to not cause problems are preserved. */
12218
12219         tmp_state = crtc_state->base;
12220         scaler_state = crtc_state->scaler_state;
12221         shared_dpll = crtc_state->shared_dpll;
12222         dpll_hw_state = crtc_state->dpll_hw_state;
12223         ddi_pll_sel = crtc_state->ddi_pll_sel;
12224         force_thru = crtc_state->pch_pfit.force_thru;
12225
12226         memset(crtc_state, 0, sizeof *crtc_state);
12227
12228         crtc_state->base = tmp_state;
12229         crtc_state->scaler_state = scaler_state;
12230         crtc_state->shared_dpll = shared_dpll;
12231         crtc_state->dpll_hw_state = dpll_hw_state;
12232         crtc_state->ddi_pll_sel = ddi_pll_sel;
12233         crtc_state->pch_pfit.force_thru = force_thru;
12234 }
12235
12236 static int
12237 intel_modeset_pipe_config(struct drm_crtc *crtc,
12238                           struct intel_crtc_state *pipe_config)
12239 {
12240         struct drm_atomic_state *state = pipe_config->base.state;
12241         struct intel_encoder *encoder;
12242         struct drm_connector *connector;
12243         struct drm_connector_state *connector_state;
12244         int base_bpp, ret = -EINVAL;
12245         int i;
12246         bool retry = true;
12247
12248         clear_intel_crtc_state(pipe_config);
12249
12250         pipe_config->cpu_transcoder =
12251                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12252
12253         /*
12254          * Sanitize sync polarity flags based on requested ones. If neither
12255          * positive or negative polarity is requested, treat this as meaning
12256          * negative polarity.
12257          */
12258         if (!(pipe_config->base.adjusted_mode.flags &
12259               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12260                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12261
12262         if (!(pipe_config->base.adjusted_mode.flags &
12263               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12264                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12265
12266         base_bpp = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12267                                              pipe_config);
12268         if (base_bpp < 0)
12269                 goto fail;
12270
12271         /*
12272          * Determine the real pipe dimensions. Note that stereo modes can
12273          * increase the actual pipe size due to the frame doubling and
12274          * insertion of additional space for blanks between the frame. This
12275          * is stored in the crtc timings. We use the requested mode to do this
12276          * computation to clearly distinguish it from the adjusted mode, which
12277          * can be changed by the connectors in the below retry loop.
12278          */
12279         drm_crtc_get_hv_timing(&pipe_config->base.mode,
12280                                &pipe_config->pipe_src_w,
12281                                &pipe_config->pipe_src_h);
12282
12283 encoder_retry:
12284         /* Ensure the port clock defaults are reset when retrying. */
12285         pipe_config->port_clock = 0;
12286         pipe_config->pixel_multiplier = 1;
12287
12288         /* Fill in default crtc timings, allow encoders to overwrite them. */
12289         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
12290                               CRTC_STEREO_DOUBLE);
12291
12292         /* Pass our mode to the connectors and the CRTC to give them a chance to
12293          * adjust it according to limitations or connector properties, and also
12294          * a chance to reject the mode entirely.
12295          */
12296         for_each_connector_in_state(state, connector, connector_state, i) {
12297                 if (connector_state->crtc != crtc)
12298                         continue;
12299
12300                 encoder = to_intel_encoder(connector_state->best_encoder);
12301
12302                 if (!(encoder->compute_config(encoder, pipe_config))) {
12303                         DRM_DEBUG_KMS("Encoder config failure\n");
12304                         goto fail;
12305                 }
12306         }
12307
12308         /* Set default port clock if not overwritten by the encoder. Needs to be
12309          * done afterwards in case the encoder adjusts the mode. */
12310         if (!pipe_config->port_clock)
12311                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
12312                         * pipe_config->pixel_multiplier;
12313
12314         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
12315         if (ret < 0) {
12316                 DRM_DEBUG_KMS("CRTC fixup failed\n");
12317                 goto fail;
12318         }
12319
12320         if (ret == RETRY) {
12321                 if (WARN(!retry, "loop in pipe configuration computation\n")) {
12322                         ret = -EINVAL;
12323                         goto fail;
12324                 }
12325
12326                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
12327                 retry = false;
12328                 goto encoder_retry;
12329         }
12330
12331         /* Dithering seems to not pass-through bits correctly when it should, so
12332          * only enable it on 6bpc panels. */
12333         pipe_config->dither = pipe_config->pipe_bpp == 6*3;
12334         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
12335                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
12336
12337 fail:
12338         return ret;
12339 }
12340
12341 static void
12342 intel_modeset_update_crtc_state(struct drm_atomic_state *state)
12343 {
12344         struct drm_crtc *crtc;
12345         struct drm_crtc_state *crtc_state;
12346         int i;
12347
12348         /* Double check state. */
12349         for_each_crtc_in_state(state, crtc, crtc_state, i) {
12350                 to_intel_crtc(crtc)->config = to_intel_crtc_state(crtc->state);
12351
12352                 /* Update hwmode for vblank functions */
12353                 if (crtc->state->active)
12354                         crtc->hwmode = crtc->state->adjusted_mode;
12355                 else
12356                         crtc->hwmode.crtc_clock = 0;
12357         }
12358 }
12359
12360 static bool intel_fuzzy_clock_check(int clock1, int clock2)
12361 {
12362         int diff;
12363
12364         if (clock1 == clock2)
12365                 return true;
12366
12367         if (!clock1 || !clock2)
12368                 return false;
12369
12370         diff = abs(clock1 - clock2);
12371
12372         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
12373                 return true;
12374
12375         return false;
12376 }
12377
12378 #define for_each_intel_crtc_masked(dev, mask, intel_crtc) \
12379         list_for_each_entry((intel_crtc), \
12380                             &(dev)->mode_config.crtc_list, \
12381                             base.head) \
12382                 if (mask & (1 <<(intel_crtc)->pipe))
12383
12384 static bool
12385 intel_compare_m_n(unsigned int m, unsigned int n,
12386                   unsigned int m2, unsigned int n2,
12387                   bool exact)
12388 {
12389         if (m == m2 && n == n2)
12390                 return true;
12391
12392         if (exact || !m || !n || !m2 || !n2)
12393                 return false;
12394
12395         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
12396
12397         if (m > m2) {
12398                 while (m > m2) {
12399                         m2 <<= 1;
12400                         n2 <<= 1;
12401                 }
12402         } else if (m < m2) {
12403                 while (m < m2) {
12404                         m <<= 1;
12405                         n <<= 1;
12406                 }
12407         }
12408
12409         return m == m2 && n == n2;
12410 }
12411
12412 static bool
12413 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
12414                        struct intel_link_m_n *m2_n2,
12415                        bool adjust)
12416 {
12417         if (m_n->tu == m2_n2->tu &&
12418             intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
12419                               m2_n2->gmch_m, m2_n2->gmch_n, !adjust) &&
12420             intel_compare_m_n(m_n->link_m, m_n->link_n,
12421                               m2_n2->link_m, m2_n2->link_n, !adjust)) {
12422                 if (adjust)
12423                         *m2_n2 = *m_n;
12424
12425                 return true;
12426         }
12427
12428         return false;
12429 }
12430
12431 static bool
12432 intel_pipe_config_compare(struct drm_device *dev,
12433                           struct intel_crtc_state *current_config,
12434                           struct intel_crtc_state *pipe_config,
12435                           bool adjust)
12436 {
12437         bool ret = true;
12438
12439 #define INTEL_ERR_OR_DBG_KMS(fmt, ...) \
12440         do { \
12441                 if (!adjust) \
12442                         DRM_ERROR(fmt, ##__VA_ARGS__); \
12443                 else \
12444                         DRM_DEBUG_KMS(fmt, ##__VA_ARGS__); \
12445         } while (0)
12446
12447 #define PIPE_CONF_CHECK_X(name) \
12448         if (current_config->name != pipe_config->name) { \
12449                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12450                           "(expected 0x%08x, found 0x%08x)\n", \
12451                           current_config->name, \
12452                           pipe_config->name); \
12453                 ret = false; \
12454         }
12455
12456 #define PIPE_CONF_CHECK_I(name) \
12457         if (current_config->name != pipe_config->name) { \
12458                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12459                           "(expected %i, found %i)\n", \
12460                           current_config->name, \
12461                           pipe_config->name); \
12462                 ret = false; \
12463         }
12464
12465 #define PIPE_CONF_CHECK_M_N(name) \
12466         if (!intel_compare_link_m_n(&current_config->name, \
12467                                     &pipe_config->name,\
12468                                     adjust)) { \
12469                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12470                           "(expected tu %i gmch %i/%i link %i/%i, " \
12471                           "found tu %i, gmch %i/%i link %i/%i)\n", \
12472                           current_config->name.tu, \
12473                           current_config->name.gmch_m, \
12474                           current_config->name.gmch_n, \
12475                           current_config->name.link_m, \
12476                           current_config->name.link_n, \
12477                           pipe_config->name.tu, \
12478                           pipe_config->name.gmch_m, \
12479                           pipe_config->name.gmch_n, \
12480                           pipe_config->name.link_m, \
12481                           pipe_config->name.link_n); \
12482                 ret = false; \
12483         }
12484
12485 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) \
12486         if (!intel_compare_link_m_n(&current_config->name, \
12487                                     &pipe_config->name, adjust) && \
12488             !intel_compare_link_m_n(&current_config->alt_name, \
12489                                     &pipe_config->name, adjust)) { \
12490                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12491                           "(expected tu %i gmch %i/%i link %i/%i, " \
12492                           "or tu %i gmch %i/%i link %i/%i, " \
12493                           "found tu %i, gmch %i/%i link %i/%i)\n", \
12494                           current_config->name.tu, \
12495                           current_config->name.gmch_m, \
12496                           current_config->name.gmch_n, \
12497                           current_config->name.link_m, \
12498                           current_config->name.link_n, \
12499                           current_config->alt_name.tu, \
12500                           current_config->alt_name.gmch_m, \
12501                           current_config->alt_name.gmch_n, \
12502                           current_config->alt_name.link_m, \
12503                           current_config->alt_name.link_n, \
12504                           pipe_config->name.tu, \
12505                           pipe_config->name.gmch_m, \
12506                           pipe_config->name.gmch_n, \
12507                           pipe_config->name.link_m, \
12508                           pipe_config->name.link_n); \
12509                 ret = false; \
12510         }
12511
12512 /* This is required for BDW+ where there is only one set of registers for
12513  * switching between high and low RR.
12514  * This macro can be used whenever a comparison has to be made between one
12515  * hw state and multiple sw state variables.
12516  */
12517 #define PIPE_CONF_CHECK_I_ALT(name, alt_name) \
12518         if ((current_config->name != pipe_config->name) && \
12519                 (current_config->alt_name != pipe_config->name)) { \
12520                         INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12521                                   "(expected %i or %i, found %i)\n", \
12522                                   current_config->name, \
12523                                   current_config->alt_name, \
12524                                   pipe_config->name); \
12525                         ret = false; \
12526         }
12527
12528 #define PIPE_CONF_CHECK_FLAGS(name, mask)       \
12529         if ((current_config->name ^ pipe_config->name) & (mask)) { \
12530                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name "(" #mask ") " \
12531                           "(expected %i, found %i)\n", \
12532                           current_config->name & (mask), \
12533                           pipe_config->name & (mask)); \
12534                 ret = false; \
12535         }
12536
12537 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) \
12538         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
12539                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12540                           "(expected %i, found %i)\n", \
12541                           current_config->name, \
12542                           pipe_config->name); \
12543                 ret = false; \
12544         }
12545
12546 #define PIPE_CONF_QUIRK(quirk)  \
12547         ((current_config->quirks | pipe_config->quirks) & (quirk))
12548
12549         PIPE_CONF_CHECK_I(cpu_transcoder);
12550
12551         PIPE_CONF_CHECK_I(has_pch_encoder);
12552         PIPE_CONF_CHECK_I(fdi_lanes);
12553         PIPE_CONF_CHECK_M_N(fdi_m_n);
12554
12555         PIPE_CONF_CHECK_I(has_dp_encoder);
12556         PIPE_CONF_CHECK_I(lane_count);
12557
12558         if (INTEL_INFO(dev)->gen < 8) {
12559                 PIPE_CONF_CHECK_M_N(dp_m_n);
12560
12561                 if (current_config->has_drrs)
12562                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
12563         } else
12564                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
12565
12566         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
12567         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
12568         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
12569         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
12570         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
12571         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
12572
12573         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
12574         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
12575         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
12576         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
12577         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
12578         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
12579
12580         PIPE_CONF_CHECK_I(pixel_multiplier);
12581         PIPE_CONF_CHECK_I(has_hdmi_sink);
12582         if ((INTEL_INFO(dev)->gen < 8 && !IS_HASWELL(dev)) ||
12583             IS_VALLEYVIEW(dev))
12584                 PIPE_CONF_CHECK_I(limited_color_range);
12585         PIPE_CONF_CHECK_I(has_infoframe);
12586
12587         PIPE_CONF_CHECK_I(has_audio);
12588
12589         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12590                               DRM_MODE_FLAG_INTERLACE);
12591
12592         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
12593                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12594                                       DRM_MODE_FLAG_PHSYNC);
12595                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12596                                       DRM_MODE_FLAG_NHSYNC);
12597                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12598                                       DRM_MODE_FLAG_PVSYNC);
12599                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12600                                       DRM_MODE_FLAG_NVSYNC);
12601         }
12602
12603         PIPE_CONF_CHECK_X(gmch_pfit.control);
12604         /* pfit ratios are autocomputed by the hw on gen4+ */
12605         if (INTEL_INFO(dev)->gen < 4)
12606                 PIPE_CONF_CHECK_I(gmch_pfit.pgm_ratios);
12607         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
12608
12609         if (!adjust) {
12610                 PIPE_CONF_CHECK_I(pipe_src_w);
12611                 PIPE_CONF_CHECK_I(pipe_src_h);
12612
12613                 PIPE_CONF_CHECK_I(pch_pfit.enabled);
12614                 if (current_config->pch_pfit.enabled) {
12615                         PIPE_CONF_CHECK_X(pch_pfit.pos);
12616                         PIPE_CONF_CHECK_X(pch_pfit.size);
12617                 }
12618
12619                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
12620         }
12621
12622         /* BDW+ don't expose a synchronous way to read the state */
12623         if (IS_HASWELL(dev))
12624                 PIPE_CONF_CHECK_I(ips_enabled);
12625
12626         PIPE_CONF_CHECK_I(double_wide);
12627
12628         PIPE_CONF_CHECK_X(ddi_pll_sel);
12629
12630         PIPE_CONF_CHECK_I(shared_dpll);
12631         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
12632         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
12633         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
12634         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
12635         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
12636         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
12637         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
12638         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
12639         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
12640
12641         if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5)
12642                 PIPE_CONF_CHECK_I(pipe_bpp);
12643
12644         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
12645         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
12646
12647 #undef PIPE_CONF_CHECK_X
12648 #undef PIPE_CONF_CHECK_I
12649 #undef PIPE_CONF_CHECK_I_ALT
12650 #undef PIPE_CONF_CHECK_FLAGS
12651 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
12652 #undef PIPE_CONF_QUIRK
12653 #undef INTEL_ERR_OR_DBG_KMS
12654
12655         return ret;
12656 }
12657
12658 static void check_wm_state(struct drm_device *dev)
12659 {
12660         struct drm_i915_private *dev_priv = dev->dev_private;
12661         struct skl_ddb_allocation hw_ddb, *sw_ddb;
12662         struct intel_crtc *intel_crtc;
12663         int plane;
12664
12665         if (INTEL_INFO(dev)->gen < 9)
12666                 return;
12667
12668         skl_ddb_get_hw_state(dev_priv, &hw_ddb);
12669         sw_ddb = &dev_priv->wm.skl_hw.ddb;
12670
12671         for_each_intel_crtc(dev, intel_crtc) {
12672                 struct skl_ddb_entry *hw_entry, *sw_entry;
12673                 const enum pipe pipe = intel_crtc->pipe;
12674
12675                 if (!intel_crtc->active)
12676                         continue;
12677
12678                 /* planes */
12679                 for_each_plane(dev_priv, pipe, plane) {
12680                         hw_entry = &hw_ddb.plane[pipe][plane];
12681                         sw_entry = &sw_ddb->plane[pipe][plane];
12682
12683                         if (skl_ddb_entry_equal(hw_entry, sw_entry))
12684                                 continue;
12685
12686                         DRM_ERROR("mismatch in DDB state pipe %c plane %d "
12687                                   "(expected (%u,%u), found (%u,%u))\n",
12688                                   pipe_name(pipe), plane + 1,
12689                                   sw_entry->start, sw_entry->end,
12690                                   hw_entry->start, hw_entry->end);
12691                 }
12692
12693                 /* cursor */
12694                 hw_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
12695                 sw_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
12696
12697                 if (skl_ddb_entry_equal(hw_entry, sw_entry))
12698                         continue;
12699
12700                 DRM_ERROR("mismatch in DDB state pipe %c cursor "
12701                           "(expected (%u,%u), found (%u,%u))\n",
12702                           pipe_name(pipe),
12703                           sw_entry->start, sw_entry->end,
12704                           hw_entry->start, hw_entry->end);
12705         }
12706 }
12707
12708 static void
12709 check_connector_state(struct drm_device *dev,
12710                       struct drm_atomic_state *old_state)
12711 {
12712         struct drm_connector_state *old_conn_state;
12713         struct drm_connector *connector;
12714         int i;
12715
12716         for_each_connector_in_state(old_state, connector, old_conn_state, i) {
12717                 struct drm_encoder *encoder = connector->encoder;
12718                 struct drm_connector_state *state = connector->state;
12719
12720                 /* This also checks the encoder/connector hw state with the
12721                  * ->get_hw_state callbacks. */
12722                 intel_connector_check_state(to_intel_connector(connector));
12723
12724                 I915_STATE_WARN(state->best_encoder != encoder,
12725                      "connector's atomic encoder doesn't match legacy encoder\n");
12726         }
12727 }
12728
12729 static void
12730 check_encoder_state(struct drm_device *dev)
12731 {
12732         struct intel_encoder *encoder;
12733         struct intel_connector *connector;
12734
12735         for_each_intel_encoder(dev, encoder) {
12736                 bool enabled = false;
12737                 enum pipe pipe;
12738
12739                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
12740                               encoder->base.base.id,
12741                               encoder->base.name);
12742
12743                 for_each_intel_connector(dev, connector) {
12744                         if (connector->base.state->best_encoder != &encoder->base)
12745                                 continue;
12746                         enabled = true;
12747
12748                         I915_STATE_WARN(connector->base.state->crtc !=
12749                                         encoder->base.crtc,
12750                              "connector's crtc doesn't match encoder crtc\n");
12751                 }
12752
12753                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
12754                      "encoder's enabled state mismatch "
12755                      "(expected %i, found %i)\n",
12756                      !!encoder->base.crtc, enabled);
12757
12758                 if (!encoder->base.crtc) {
12759                         bool active;
12760
12761                         active = encoder->get_hw_state(encoder, &pipe);
12762                         I915_STATE_WARN(active,
12763                              "encoder detached but still enabled on pipe %c.\n",
12764                              pipe_name(pipe));
12765                 }
12766         }
12767 }
12768
12769 static void
12770 check_crtc_state(struct drm_device *dev, struct drm_atomic_state *old_state)
12771 {
12772         struct drm_i915_private *dev_priv = dev->dev_private;
12773         struct intel_encoder *encoder;
12774         struct drm_crtc_state *old_crtc_state;
12775         struct drm_crtc *crtc;
12776         int i;
12777
12778         for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) {
12779                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12780                 struct intel_crtc_state *pipe_config, *sw_config;
12781                 bool active;
12782
12783                 if (!needs_modeset(crtc->state) &&
12784                     !to_intel_crtc_state(crtc->state)->update_pipe)
12785                         continue;
12786
12787                 __drm_atomic_helper_crtc_destroy_state(crtc, old_crtc_state);
12788                 pipe_config = to_intel_crtc_state(old_crtc_state);
12789                 memset(pipe_config, 0, sizeof(*pipe_config));
12790                 pipe_config->base.crtc = crtc;
12791                 pipe_config->base.state = old_state;
12792
12793                 DRM_DEBUG_KMS("[CRTC:%d]\n",
12794                               crtc->base.id);
12795
12796                 active = dev_priv->display.get_pipe_config(intel_crtc,
12797                                                            pipe_config);
12798
12799                 /* hw state is inconsistent with the pipe quirk */
12800                 if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
12801                     (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
12802                         active = crtc->state->active;
12803
12804                 I915_STATE_WARN(crtc->state->active != active,
12805                      "crtc active state doesn't match with hw state "
12806                      "(expected %i, found %i)\n", crtc->state->active, active);
12807
12808                 I915_STATE_WARN(intel_crtc->active != crtc->state->active,
12809                      "transitional active state does not match atomic hw state "
12810                      "(expected %i, found %i)\n", crtc->state->active, intel_crtc->active);
12811
12812                 for_each_encoder_on_crtc(dev, crtc, encoder) {
12813                         enum pipe pipe;
12814
12815                         active = encoder->get_hw_state(encoder, &pipe);
12816                         I915_STATE_WARN(active != crtc->state->active,
12817                                 "[ENCODER:%i] active %i with crtc active %i\n",
12818                                 encoder->base.base.id, active, crtc->state->active);
12819
12820                         I915_STATE_WARN(active && intel_crtc->pipe != pipe,
12821                                         "Encoder connected to wrong pipe %c\n",
12822                                         pipe_name(pipe));
12823
12824                         if (active)
12825                                 encoder->get_config(encoder, pipe_config);
12826                 }
12827
12828                 if (!crtc->state->active)
12829                         continue;
12830
12831                 sw_config = to_intel_crtc_state(crtc->state);
12832                 if (!intel_pipe_config_compare(dev, sw_config,
12833                                                pipe_config, false)) {
12834                         I915_STATE_WARN(1, "pipe state doesn't match!\n");
12835                         intel_dump_pipe_config(intel_crtc, pipe_config,
12836                                                "[hw state]");
12837                         intel_dump_pipe_config(intel_crtc, sw_config,
12838                                                "[sw state]");
12839                 }
12840         }
12841 }
12842
12843 static void
12844 check_shared_dpll_state(struct drm_device *dev)
12845 {
12846         struct drm_i915_private *dev_priv = dev->dev_private;
12847         struct intel_crtc *crtc;
12848         struct intel_dpll_hw_state dpll_hw_state;
12849         int i;
12850
12851         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
12852                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
12853                 int enabled_crtcs = 0, active_crtcs = 0;
12854                 bool active;
12855
12856                 memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
12857
12858                 DRM_DEBUG_KMS("%s\n", pll->name);
12859
12860                 active = pll->get_hw_state(dev_priv, pll, &dpll_hw_state);
12861
12862                 I915_STATE_WARN(pll->active > hweight32(pll->config.crtc_mask),
12863                      "more active pll users than references: %i vs %i\n",
12864                      pll->active, hweight32(pll->config.crtc_mask));
12865                 I915_STATE_WARN(pll->active && !pll->on,
12866                      "pll in active use but not on in sw tracking\n");
12867                 I915_STATE_WARN(pll->on && !pll->active,
12868                      "pll in on but not on in use in sw tracking\n");
12869                 I915_STATE_WARN(pll->on != active,
12870                      "pll on state mismatch (expected %i, found %i)\n",
12871                      pll->on, active);
12872
12873                 for_each_intel_crtc(dev, crtc) {
12874                         if (crtc->base.state->enable && intel_crtc_to_shared_dpll(crtc) == pll)
12875                                 enabled_crtcs++;
12876                         if (crtc->active && intel_crtc_to_shared_dpll(crtc) == pll)
12877                                 active_crtcs++;
12878                 }
12879                 I915_STATE_WARN(pll->active != active_crtcs,
12880                      "pll active crtcs mismatch (expected %i, found %i)\n",
12881                      pll->active, active_crtcs);
12882                 I915_STATE_WARN(hweight32(pll->config.crtc_mask) != enabled_crtcs,
12883                      "pll enabled crtcs mismatch (expected %i, found %i)\n",
12884                      hweight32(pll->config.crtc_mask), enabled_crtcs);
12885
12886                 I915_STATE_WARN(pll->on && memcmp(&pll->config.hw_state, &dpll_hw_state,
12887                                        sizeof(dpll_hw_state)),
12888                      "pll hw state mismatch\n");
12889         }
12890 }
12891
12892 static void
12893 intel_modeset_check_state(struct drm_device *dev,
12894                           struct drm_atomic_state *old_state)
12895 {
12896         check_wm_state(dev);
12897         check_connector_state(dev, old_state);
12898         check_encoder_state(dev);
12899         check_crtc_state(dev, old_state);
12900         check_shared_dpll_state(dev);
12901 }
12902
12903 void ironlake_check_encoder_dotclock(const struct intel_crtc_state *pipe_config,
12904                                      int dotclock)
12905 {
12906         /*
12907          * FDI already provided one idea for the dotclock.
12908          * Yell if the encoder disagrees.
12909          */
12910         WARN(!intel_fuzzy_clock_check(pipe_config->base.adjusted_mode.crtc_clock, dotclock),
12911              "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
12912              pipe_config->base.adjusted_mode.crtc_clock, dotclock);
12913 }
12914
12915 static void update_scanline_offset(struct intel_crtc *crtc)
12916 {
12917         struct drm_device *dev = crtc->base.dev;
12918
12919         /*
12920          * The scanline counter increments at the leading edge of hsync.
12921          *
12922          * On most platforms it starts counting from vtotal-1 on the
12923          * first active line. That means the scanline counter value is
12924          * always one less than what we would expect. Ie. just after
12925          * start of vblank, which also occurs at start of hsync (on the
12926          * last active line), the scanline counter will read vblank_start-1.
12927          *
12928          * On gen2 the scanline counter starts counting from 1 instead
12929          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
12930          * to keep the value positive), instead of adding one.
12931          *
12932          * On HSW+ the behaviour of the scanline counter depends on the output
12933          * type. For DP ports it behaves like most other platforms, but on HDMI
12934          * there's an extra 1 line difference. So we need to add two instead of
12935          * one to the value.
12936          */
12937         if (IS_GEN2(dev)) {
12938                 const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
12939                 int vtotal;
12940
12941                 vtotal = adjusted_mode->crtc_vtotal;
12942                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
12943                         vtotal /= 2;
12944
12945                 crtc->scanline_offset = vtotal - 1;
12946         } else if (HAS_DDI(dev) &&
12947                    intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI)) {
12948                 crtc->scanline_offset = 2;
12949         } else
12950                 crtc->scanline_offset = 1;
12951 }
12952
12953 static void intel_modeset_clear_plls(struct drm_atomic_state *state)
12954 {
12955         struct drm_device *dev = state->dev;
12956         struct drm_i915_private *dev_priv = to_i915(dev);
12957         struct intel_shared_dpll_config *shared_dpll = NULL;
12958         struct intel_crtc *intel_crtc;
12959         struct intel_crtc_state *intel_crtc_state;
12960         struct drm_crtc *crtc;
12961         struct drm_crtc_state *crtc_state;
12962         int i;
12963
12964         if (!dev_priv->display.crtc_compute_clock)
12965                 return;
12966
12967         for_each_crtc_in_state(state, crtc, crtc_state, i) {
12968                 int dpll;
12969
12970                 intel_crtc = to_intel_crtc(crtc);
12971                 intel_crtc_state = to_intel_crtc_state(crtc_state);
12972                 dpll = intel_crtc_state->shared_dpll;
12973
12974                 if (!needs_modeset(crtc_state) || dpll == DPLL_ID_PRIVATE)
12975                         continue;
12976
12977                 intel_crtc_state->shared_dpll = DPLL_ID_PRIVATE;
12978
12979                 if (!shared_dpll)
12980                         shared_dpll = intel_atomic_get_shared_dpll_state(state);
12981
12982                 shared_dpll[dpll].crtc_mask &= ~(1 << intel_crtc->pipe);
12983         }
12984 }
12985
12986 /*
12987  * This implements the workaround described in the "notes" section of the mode
12988  * set sequence documentation. When going from no pipes or single pipe to
12989  * multiple pipes, and planes are enabled after the pipe, we need to wait at
12990  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
12991  */
12992 static int haswell_mode_set_planes_workaround(struct drm_atomic_state *state)
12993 {
12994         struct drm_crtc_state *crtc_state;
12995         struct intel_crtc *intel_crtc;
12996         struct drm_crtc *crtc;
12997         struct intel_crtc_state *first_crtc_state = NULL;
12998         struct intel_crtc_state *other_crtc_state = NULL;
12999         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
13000         int i;
13001
13002         /* look at all crtc's that are going to be enabled in during modeset */
13003         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13004                 intel_crtc = to_intel_crtc(crtc);
13005
13006                 if (!crtc_state->active || !needs_modeset(crtc_state))
13007                         continue;
13008
13009                 if (first_crtc_state) {
13010                         other_crtc_state = to_intel_crtc_state(crtc_state);
13011                         break;
13012                 } else {
13013                         first_crtc_state = to_intel_crtc_state(crtc_state);
13014                         first_pipe = intel_crtc->pipe;
13015                 }
13016         }
13017
13018         /* No workaround needed? */
13019         if (!first_crtc_state)
13020                 return 0;
13021
13022         /* w/a possibly needed, check how many crtc's are already enabled. */
13023         for_each_intel_crtc(state->dev, intel_crtc) {
13024                 struct intel_crtc_state *pipe_config;
13025
13026                 pipe_config = intel_atomic_get_crtc_state(state, intel_crtc);
13027                 if (IS_ERR(pipe_config))
13028                         return PTR_ERR(pipe_config);
13029
13030                 pipe_config->hsw_workaround_pipe = INVALID_PIPE;
13031
13032                 if (!pipe_config->base.active ||
13033                     needs_modeset(&pipe_config->base))
13034                         continue;
13035
13036                 /* 2 or more enabled crtcs means no need for w/a */
13037                 if (enabled_pipe != INVALID_PIPE)
13038                         return 0;
13039
13040                 enabled_pipe = intel_crtc->pipe;
13041         }
13042
13043         if (enabled_pipe != INVALID_PIPE)
13044                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
13045         else if (other_crtc_state)
13046                 other_crtc_state->hsw_workaround_pipe = first_pipe;
13047
13048         return 0;
13049 }
13050
13051 static int intel_modeset_all_pipes(struct drm_atomic_state *state)
13052 {
13053         struct drm_crtc *crtc;
13054         struct drm_crtc_state *crtc_state;
13055         int ret = 0;
13056
13057         /* add all active pipes to the state */
13058         for_each_crtc(state->dev, crtc) {
13059                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
13060                 if (IS_ERR(crtc_state))
13061                         return PTR_ERR(crtc_state);
13062
13063                 if (!crtc_state->active || needs_modeset(crtc_state))
13064                         continue;
13065
13066                 crtc_state->mode_changed = true;
13067
13068                 ret = drm_atomic_add_affected_connectors(state, crtc);
13069                 if (ret)
13070                         break;
13071
13072                 ret = drm_atomic_add_affected_planes(state, crtc);
13073                 if (ret)
13074                         break;
13075         }
13076
13077         return ret;
13078 }
13079
13080 static int intel_modeset_checks(struct drm_atomic_state *state)
13081 {
13082         struct drm_device *dev = state->dev;
13083         struct drm_i915_private *dev_priv = dev->dev_private;
13084         int ret;
13085
13086         if (!check_digital_port_conflicts(state)) {
13087                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
13088                 return -EINVAL;
13089         }
13090
13091         /*
13092          * See if the config requires any additional preparation, e.g.
13093          * to adjust global state with pipes off.  We need to do this
13094          * here so we can get the modeset_pipe updated config for the new
13095          * mode set on this crtc.  For other crtcs we need to use the
13096          * adjusted_mode bits in the crtc directly.
13097          */
13098         if (dev_priv->display.modeset_calc_cdclk) {
13099                 unsigned int cdclk;
13100
13101                 ret = dev_priv->display.modeset_calc_cdclk(state);
13102
13103                 cdclk = to_intel_atomic_state(state)->cdclk;
13104                 if (!ret && cdclk != dev_priv->cdclk_freq)
13105                         ret = intel_modeset_all_pipes(state);
13106
13107                 if (ret < 0)
13108                         return ret;
13109         } else
13110                 to_intel_atomic_state(state)->cdclk = dev_priv->cdclk_freq;
13111
13112         intel_modeset_clear_plls(state);
13113
13114         if (IS_HASWELL(dev))
13115                 return haswell_mode_set_planes_workaround(state);
13116
13117         return 0;
13118 }
13119
13120 /**
13121  * intel_atomic_check - validate state object
13122  * @dev: drm device
13123  * @state: state to validate
13124  */
13125 static int intel_atomic_check(struct drm_device *dev,
13126                               struct drm_atomic_state *state)
13127 {
13128         struct drm_crtc *crtc;
13129         struct drm_crtc_state *crtc_state;
13130         int ret, i;
13131         bool any_ms = false;
13132
13133         ret = drm_atomic_helper_check_modeset(dev, state);
13134         if (ret)
13135                 return ret;
13136
13137         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13138                 struct intel_crtc_state *pipe_config =
13139                         to_intel_crtc_state(crtc_state);
13140
13141                 memset(&to_intel_crtc(crtc)->atomic, 0,
13142                        sizeof(struct intel_crtc_atomic_commit));
13143
13144                 /* Catch I915_MODE_FLAG_INHERITED */
13145                 if (crtc_state->mode.private_flags != crtc->state->mode.private_flags)
13146                         crtc_state->mode_changed = true;
13147
13148                 if (!crtc_state->enable) {
13149                         if (needs_modeset(crtc_state))
13150                                 any_ms = true;
13151                         continue;
13152                 }
13153
13154                 if (!needs_modeset(crtc_state))
13155                         continue;
13156
13157                 /* FIXME: For only active_changed we shouldn't need to do any
13158                  * state recomputation at all. */
13159
13160                 ret = drm_atomic_add_affected_connectors(state, crtc);
13161                 if (ret)
13162                         return ret;
13163
13164                 ret = intel_modeset_pipe_config(crtc, pipe_config);
13165                 if (ret)
13166                         return ret;
13167
13168                 if (i915.fastboot &&
13169                     intel_pipe_config_compare(state->dev,
13170                                         to_intel_crtc_state(crtc->state),
13171                                         pipe_config, true)) {
13172                         crtc_state->mode_changed = false;
13173                         to_intel_crtc_state(crtc_state)->update_pipe = true;
13174                 }
13175
13176                 if (needs_modeset(crtc_state)) {
13177                         any_ms = true;
13178
13179                         ret = drm_atomic_add_affected_planes(state, crtc);
13180                         if (ret)
13181                                 return ret;
13182                 }
13183
13184                 intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
13185                                        needs_modeset(crtc_state) ?
13186                                        "[modeset]" : "[fastset]");
13187         }
13188
13189         if (any_ms) {
13190                 ret = intel_modeset_checks(state);
13191
13192                 if (ret)
13193                         return ret;
13194         } else
13195                 to_intel_atomic_state(state)->cdclk =
13196                         to_i915(state->dev)->cdclk_freq;
13197
13198         return drm_atomic_helper_check_planes(state->dev, state);
13199 }
13200
13201 /**
13202  * intel_atomic_commit - commit validated state object
13203  * @dev: DRM device
13204  * @state: the top-level driver state object
13205  * @async: asynchronous commit
13206  *
13207  * This function commits a top-level state object that has been validated
13208  * with drm_atomic_helper_check().
13209  *
13210  * FIXME:  Atomic modeset support for i915 is not yet complete.  At the moment
13211  * we can only handle plane-related operations and do not yet support
13212  * asynchronous commit.
13213  *
13214  * RETURNS
13215  * Zero for success or -errno.
13216  */
13217 static int intel_atomic_commit(struct drm_device *dev,
13218                                struct drm_atomic_state *state,
13219                                bool async)
13220 {
13221         struct drm_i915_private *dev_priv = dev->dev_private;
13222         struct drm_crtc *crtc;
13223         struct drm_crtc_state *crtc_state;
13224         int ret = 0;
13225         int i;
13226         bool any_ms = false;
13227
13228         if (async) {
13229                 DRM_DEBUG_KMS("i915 does not yet support async commit\n");
13230                 return -EINVAL;
13231         }
13232
13233         ret = drm_atomic_helper_prepare_planes(dev, state);
13234         if (ret)
13235                 return ret;
13236
13237         drm_atomic_helper_swap_state(dev, state);
13238
13239         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13240                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13241
13242                 if (!needs_modeset(crtc->state))
13243                         continue;
13244
13245                 any_ms = true;
13246                 intel_pre_plane_update(intel_crtc);
13247
13248                 if (crtc_state->active) {
13249                         intel_crtc_disable_planes(crtc, crtc_state->plane_mask);
13250                         dev_priv->display.crtc_disable(crtc);
13251                         intel_crtc->active = false;
13252                         intel_disable_shared_dpll(intel_crtc);
13253                 }
13254         }
13255
13256         /* Only after disabling all output pipelines that will be changed can we
13257          * update the the output configuration. */
13258         intel_modeset_update_crtc_state(state);
13259
13260         if (any_ms) {
13261                 intel_shared_dpll_commit(state);
13262
13263                 drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
13264                 modeset_update_crtc_power_domains(state);
13265         }
13266
13267         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
13268         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13269                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13270                 bool modeset = needs_modeset(crtc->state);
13271                 bool update_pipe = !modeset &&
13272                         to_intel_crtc_state(crtc->state)->update_pipe;
13273                 unsigned long put_domains = 0;
13274
13275                 if (modeset && crtc->state->active) {
13276                         update_scanline_offset(to_intel_crtc(crtc));
13277                         dev_priv->display.crtc_enable(crtc);
13278                 }
13279
13280                 if (update_pipe) {
13281                         put_domains = modeset_get_crtc_power_domains(crtc);
13282
13283                         /* make sure intel_modeset_check_state runs */
13284                         any_ms = true;
13285                 }
13286
13287                 if (!modeset)
13288                         intel_pre_plane_update(intel_crtc);
13289
13290                 drm_atomic_helper_commit_planes_on_crtc(crtc_state);
13291
13292                 if (put_domains)
13293                         modeset_put_power_domains(dev_priv, put_domains);
13294
13295                 intel_post_plane_update(intel_crtc);
13296         }
13297
13298         /* FIXME: add subpixel order */
13299
13300         drm_atomic_helper_wait_for_vblanks(dev, state);
13301         drm_atomic_helper_cleanup_planes(dev, state);
13302
13303         if (any_ms)
13304                 intel_modeset_check_state(dev, state);
13305
13306         drm_atomic_state_free(state);
13307
13308         return 0;
13309 }
13310
13311 void intel_crtc_restore_mode(struct drm_crtc *crtc)
13312 {
13313         struct drm_device *dev = crtc->dev;
13314         struct drm_atomic_state *state;
13315         struct drm_crtc_state *crtc_state;
13316         int ret;
13317
13318         state = drm_atomic_state_alloc(dev);
13319         if (!state) {
13320                 DRM_DEBUG_KMS("[CRTC:%d] crtc restore failed, out of memory",
13321                               crtc->base.id);
13322                 return;
13323         }
13324
13325         state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc);
13326
13327 retry:
13328         crtc_state = drm_atomic_get_crtc_state(state, crtc);
13329         ret = PTR_ERR_OR_ZERO(crtc_state);
13330         if (!ret) {
13331                 if (!crtc_state->active)
13332                         goto out;
13333
13334                 crtc_state->mode_changed = true;
13335                 ret = drm_atomic_commit(state);
13336         }
13337
13338         if (ret == -EDEADLK) {
13339                 drm_atomic_state_clear(state);
13340                 drm_modeset_backoff(state->acquire_ctx);
13341                 goto retry;
13342         }
13343
13344         if (ret)
13345 out:
13346                 drm_atomic_state_free(state);
13347 }
13348
13349 #undef for_each_intel_crtc_masked
13350
13351 static const struct drm_crtc_funcs intel_crtc_funcs = {
13352         .gamma_set = intel_crtc_gamma_set,
13353         .set_config = drm_atomic_helper_set_config,
13354         .destroy = intel_crtc_destroy,
13355         .page_flip = intel_crtc_page_flip,
13356         .atomic_duplicate_state = intel_crtc_duplicate_state,
13357         .atomic_destroy_state = intel_crtc_destroy_state,
13358 };
13359
13360 static bool ibx_pch_dpll_get_hw_state(struct drm_i915_private *dev_priv,
13361                                       struct intel_shared_dpll *pll,
13362                                       struct intel_dpll_hw_state *hw_state)
13363 {
13364         uint32_t val;
13365
13366         if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_PLLS))
13367                 return false;
13368
13369         val = I915_READ(PCH_DPLL(pll->id));
13370         hw_state->dpll = val;
13371         hw_state->fp0 = I915_READ(PCH_FP0(pll->id));
13372         hw_state->fp1 = I915_READ(PCH_FP1(pll->id));
13373
13374         return val & DPLL_VCO_ENABLE;
13375 }
13376
13377 static void ibx_pch_dpll_mode_set(struct drm_i915_private *dev_priv,
13378                                   struct intel_shared_dpll *pll)
13379 {
13380         I915_WRITE(PCH_FP0(pll->id), pll->config.hw_state.fp0);
13381         I915_WRITE(PCH_FP1(pll->id), pll->config.hw_state.fp1);
13382 }
13383
13384 static void ibx_pch_dpll_enable(struct drm_i915_private *dev_priv,
13385                                 struct intel_shared_dpll *pll)
13386 {
13387         /* PCH refclock must be enabled first */
13388         ibx_assert_pch_refclk_enabled(dev_priv);
13389
13390         I915_WRITE(PCH_DPLL(pll->id), pll->config.hw_state.dpll);
13391
13392         /* Wait for the clocks to stabilize. */
13393         POSTING_READ(PCH_DPLL(pll->id));
13394         udelay(150);
13395
13396         /* The pixel multiplier can only be updated once the
13397          * DPLL is enabled and the clocks are stable.
13398          *
13399          * So write it again.
13400          */
13401         I915_WRITE(PCH_DPLL(pll->id), pll->config.hw_state.dpll);
13402         POSTING_READ(PCH_DPLL(pll->id));
13403         udelay(200);
13404 }
13405
13406 static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv,
13407                                  struct intel_shared_dpll *pll)
13408 {
13409         struct drm_device *dev = dev_priv->dev;
13410         struct intel_crtc *crtc;
13411
13412         /* Make sure no transcoder isn't still depending on us. */
13413         for_each_intel_crtc(dev, crtc) {
13414                 if (intel_crtc_to_shared_dpll(crtc) == pll)
13415                         assert_pch_transcoder_disabled(dev_priv, crtc->pipe);
13416         }
13417
13418         I915_WRITE(PCH_DPLL(pll->id), 0);
13419         POSTING_READ(PCH_DPLL(pll->id));
13420         udelay(200);
13421 }
13422
13423 static char *ibx_pch_dpll_names[] = {
13424         "PCH DPLL A",
13425         "PCH DPLL B",
13426 };
13427
13428 static void ibx_pch_dpll_init(struct drm_device *dev)
13429 {
13430         struct drm_i915_private *dev_priv = dev->dev_private;
13431         int i;
13432
13433         dev_priv->num_shared_dpll = 2;
13434
13435         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
13436                 dev_priv->shared_dplls[i].id = i;
13437                 dev_priv->shared_dplls[i].name = ibx_pch_dpll_names[i];
13438                 dev_priv->shared_dplls[i].mode_set = ibx_pch_dpll_mode_set;
13439                 dev_priv->shared_dplls[i].enable = ibx_pch_dpll_enable;
13440                 dev_priv->shared_dplls[i].disable = ibx_pch_dpll_disable;
13441                 dev_priv->shared_dplls[i].get_hw_state =
13442                         ibx_pch_dpll_get_hw_state;
13443         }
13444 }
13445
13446 static void intel_shared_dpll_init(struct drm_device *dev)
13447 {
13448         struct drm_i915_private *dev_priv = dev->dev_private;
13449
13450         if (HAS_DDI(dev))
13451                 intel_ddi_pll_init(dev);
13452         else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
13453                 ibx_pch_dpll_init(dev);
13454         else
13455                 dev_priv->num_shared_dpll = 0;
13456
13457         BUG_ON(dev_priv->num_shared_dpll > I915_NUM_PLLS);
13458 }
13459
13460 /**
13461  * intel_prepare_plane_fb - Prepare fb for usage on plane
13462  * @plane: drm plane to prepare for
13463  * @fb: framebuffer to prepare for presentation
13464  *
13465  * Prepares a framebuffer for usage on a display plane.  Generally this
13466  * involves pinning the underlying object and updating the frontbuffer tracking
13467  * bits.  Some older platforms need special physical address handling for
13468  * cursor planes.
13469  *
13470  * Returns 0 on success, negative error code on failure.
13471  */
13472 int
13473 intel_prepare_plane_fb(struct drm_plane *plane,
13474                        const struct drm_plane_state *new_state)
13475 {
13476         struct drm_device *dev = plane->dev;
13477         struct drm_framebuffer *fb = new_state->fb;
13478         struct intel_plane *intel_plane = to_intel_plane(plane);
13479         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13480         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->fb);
13481         int ret = 0;
13482
13483         if (!obj)
13484                 return 0;
13485
13486         mutex_lock(&dev->struct_mutex);
13487
13488         if (plane->type == DRM_PLANE_TYPE_CURSOR &&
13489             INTEL_INFO(dev)->cursor_needs_physical) {
13490                 int align = IS_I830(dev) ? 16 * 1024 : 256;
13491                 ret = i915_gem_object_attach_phys(obj, align);
13492                 if (ret)
13493                         DRM_DEBUG_KMS("failed to attach phys object\n");
13494         } else {
13495                 ret = intel_pin_and_fence_fb_obj(plane, fb, new_state, NULL, NULL);
13496         }
13497
13498         if (ret == 0)
13499                 i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit);
13500
13501         mutex_unlock(&dev->struct_mutex);
13502
13503         return ret;
13504 }
13505
13506 /**
13507  * intel_cleanup_plane_fb - Cleans up an fb after plane use
13508  * @plane: drm plane to clean up for
13509  * @fb: old framebuffer that was on plane
13510  *
13511  * Cleans up a framebuffer that has just been removed from a plane.
13512  */
13513 void
13514 intel_cleanup_plane_fb(struct drm_plane *plane,
13515                        const struct drm_plane_state *old_state)
13516 {
13517         struct drm_device *dev = plane->dev;
13518         struct drm_i915_gem_object *obj = intel_fb_obj(old_state->fb);
13519
13520         if (!obj)
13521                 return;
13522
13523         if (plane->type != DRM_PLANE_TYPE_CURSOR ||
13524             !INTEL_INFO(dev)->cursor_needs_physical) {
13525                 mutex_lock(&dev->struct_mutex);
13526                 intel_unpin_fb_obj(old_state->fb, old_state);
13527                 mutex_unlock(&dev->struct_mutex);
13528         }
13529 }
13530
13531 int
13532 skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state)
13533 {
13534         int max_scale;
13535         struct drm_device *dev;
13536         struct drm_i915_private *dev_priv;
13537         int crtc_clock, cdclk;
13538
13539         if (!intel_crtc || !crtc_state)
13540                 return DRM_PLANE_HELPER_NO_SCALING;
13541
13542         dev = intel_crtc->base.dev;
13543         dev_priv = dev->dev_private;
13544         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
13545         cdclk = to_intel_atomic_state(crtc_state->base.state)->cdclk;
13546
13547         if (!crtc_clock || !cdclk)
13548                 return DRM_PLANE_HELPER_NO_SCALING;
13549
13550         /*
13551          * skl max scale is lower of:
13552          *    close to 3 but not 3, -1 is for that purpose
13553          *            or
13554          *    cdclk/crtc_clock
13555          */
13556         max_scale = min((1 << 16) * 3 - 1, (1 << 8) * ((cdclk << 8) / crtc_clock));
13557
13558         return max_scale;
13559 }
13560
13561 static int
13562 intel_check_primary_plane(struct drm_plane *plane,
13563                           struct intel_crtc_state *crtc_state,
13564                           struct intel_plane_state *state)
13565 {
13566         struct drm_crtc *crtc = state->base.crtc;
13567         struct drm_framebuffer *fb = state->base.fb;
13568         int min_scale = DRM_PLANE_HELPER_NO_SCALING;
13569         int max_scale = DRM_PLANE_HELPER_NO_SCALING;
13570         bool can_position = false;
13571
13572         if (INTEL_INFO(plane->dev)->gen >= 9) {
13573                 /* use scaler when colorkey is not required */
13574                 if (state->ckey.flags == I915_SET_COLORKEY_NONE) {
13575                         min_scale = 1;
13576                         max_scale = skl_max_scale(to_intel_crtc(crtc), crtc_state);
13577                 }
13578                 can_position = true;
13579         }
13580
13581         return drm_plane_helper_check_update(plane, crtc, fb, &state->src,
13582                                              &state->dst, &state->clip,
13583                                              min_scale, max_scale,
13584                                              can_position, true,
13585                                              &state->visible);
13586 }
13587
13588 static void
13589 intel_commit_primary_plane(struct drm_plane *plane,
13590                            struct intel_plane_state *state)
13591 {
13592         struct drm_crtc *crtc = state->base.crtc;
13593         struct drm_framebuffer *fb = state->base.fb;
13594         struct drm_device *dev = plane->dev;
13595         struct drm_i915_private *dev_priv = dev->dev_private;
13596         struct intel_crtc *intel_crtc;
13597         struct drm_rect *src = &state->src;
13598
13599         crtc = crtc ? crtc : plane->crtc;
13600         intel_crtc = to_intel_crtc(crtc);
13601
13602         plane->fb = fb;
13603         crtc->x = src->x1 >> 16;
13604         crtc->y = src->y1 >> 16;
13605
13606         if (!crtc->state->active)
13607                 return;
13608
13609         dev_priv->display.update_primary_plane(crtc, fb,
13610                                                state->src.x1 >> 16,
13611                                                state->src.y1 >> 16);
13612 }
13613
13614 static void
13615 intel_disable_primary_plane(struct drm_plane *plane,
13616                             struct drm_crtc *crtc)
13617 {
13618         struct drm_device *dev = plane->dev;
13619         struct drm_i915_private *dev_priv = dev->dev_private;
13620
13621         dev_priv->display.update_primary_plane(crtc, NULL, 0, 0);
13622 }
13623
13624 static void intel_begin_crtc_commit(struct drm_crtc *crtc,
13625                                     struct drm_crtc_state *old_crtc_state)
13626 {
13627         struct drm_device *dev = crtc->dev;
13628         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13629         struct intel_crtc_state *old_intel_state =
13630                 to_intel_crtc_state(old_crtc_state);
13631         bool modeset = needs_modeset(crtc->state);
13632
13633         if (intel_crtc->atomic.update_wm_pre)
13634                 intel_update_watermarks(crtc);
13635
13636         /* Perform vblank evasion around commit operation */
13637         if (crtc->state->active)
13638                 intel_pipe_update_start(intel_crtc);
13639
13640         if (modeset)
13641                 return;
13642
13643         if (to_intel_crtc_state(crtc->state)->update_pipe)
13644                 intel_update_pipe_config(intel_crtc, old_intel_state);
13645         else if (INTEL_INFO(dev)->gen >= 9)
13646                 skl_detach_scalers(intel_crtc);
13647 }
13648
13649 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
13650                                      struct drm_crtc_state *old_crtc_state)
13651 {
13652         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13653
13654         if (crtc->state->active)
13655                 intel_pipe_update_end(intel_crtc);
13656 }
13657
13658 /**
13659  * intel_plane_destroy - destroy a plane
13660  * @plane: plane to destroy
13661  *
13662  * Common destruction function for all types of planes (primary, cursor,
13663  * sprite).
13664  */
13665 void intel_plane_destroy(struct drm_plane *plane)
13666 {
13667         struct intel_plane *intel_plane = to_intel_plane(plane);
13668         drm_plane_cleanup(plane);
13669         kfree(intel_plane);
13670 }
13671
13672 const struct drm_plane_funcs intel_plane_funcs = {
13673         .update_plane = drm_atomic_helper_update_plane,
13674         .disable_plane = drm_atomic_helper_disable_plane,
13675         .destroy = intel_plane_destroy,
13676         .set_property = drm_atomic_helper_plane_set_property,
13677         .atomic_get_property = intel_plane_atomic_get_property,
13678         .atomic_set_property = intel_plane_atomic_set_property,
13679         .atomic_duplicate_state = intel_plane_duplicate_state,
13680         .atomic_destroy_state = intel_plane_destroy_state,
13681
13682 };
13683
13684 static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
13685                                                     int pipe)
13686 {
13687         struct intel_plane *primary;
13688         struct intel_plane_state *state;
13689         const uint32_t *intel_primary_formats;
13690         unsigned int num_formats;
13691
13692         primary = kzalloc(sizeof(*primary), GFP_KERNEL);
13693         if (primary == NULL)
13694                 return NULL;
13695
13696         state = intel_create_plane_state(&primary->base);
13697         if (!state) {
13698                 kfree(primary);
13699                 return NULL;
13700         }
13701         primary->base.state = &state->base;
13702
13703         primary->can_scale = false;
13704         primary->max_downscale = 1;
13705         if (INTEL_INFO(dev)->gen >= 9) {
13706                 primary->can_scale = true;
13707                 state->scaler_id = -1;
13708         }
13709         primary->pipe = pipe;
13710         primary->plane = pipe;
13711         primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe);
13712         primary->check_plane = intel_check_primary_plane;
13713         primary->commit_plane = intel_commit_primary_plane;
13714         primary->disable_plane = intel_disable_primary_plane;
13715         if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4)
13716                 primary->plane = !pipe;
13717
13718         if (INTEL_INFO(dev)->gen >= 9) {
13719                 intel_primary_formats = skl_primary_formats;
13720                 num_formats = ARRAY_SIZE(skl_primary_formats);
13721         } else if (INTEL_INFO(dev)->gen >= 4) {
13722                 intel_primary_formats = i965_primary_formats;
13723                 num_formats = ARRAY_SIZE(i965_primary_formats);
13724         } else {
13725                 intel_primary_formats = i8xx_primary_formats;
13726                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
13727         }
13728
13729         drm_universal_plane_init(dev, &primary->base, 0,
13730                                  &intel_plane_funcs,
13731                                  intel_primary_formats, num_formats,
13732                                  DRM_PLANE_TYPE_PRIMARY);
13733
13734         if (INTEL_INFO(dev)->gen >= 4)
13735                 intel_create_rotation_property(dev, primary);
13736
13737         drm_plane_helper_add(&primary->base, &intel_plane_helper_funcs);
13738
13739         return &primary->base;
13740 }
13741
13742 void intel_create_rotation_property(struct drm_device *dev, struct intel_plane *plane)
13743 {
13744         if (!dev->mode_config.rotation_property) {
13745                 unsigned long flags = BIT(DRM_ROTATE_0) |
13746                         BIT(DRM_ROTATE_180);
13747
13748                 if (INTEL_INFO(dev)->gen >= 9)
13749                         flags |= BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270);
13750
13751                 dev->mode_config.rotation_property =
13752                         drm_mode_create_rotation_property(dev, flags);
13753         }
13754         if (dev->mode_config.rotation_property)
13755                 drm_object_attach_property(&plane->base.base,
13756                                 dev->mode_config.rotation_property,
13757                                 plane->base.state->rotation);
13758 }
13759
13760 static int
13761 intel_check_cursor_plane(struct drm_plane *plane,
13762                          struct intel_crtc_state *crtc_state,
13763                          struct intel_plane_state *state)
13764 {
13765         struct drm_crtc *crtc = crtc_state->base.crtc;
13766         struct drm_framebuffer *fb = state->base.fb;
13767         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13768         enum pipe pipe = to_intel_plane(plane)->pipe;
13769         unsigned stride;
13770         int ret;
13771
13772         ret = drm_plane_helper_check_update(plane, crtc, fb, &state->src,
13773                                             &state->dst, &state->clip,
13774                                             DRM_PLANE_HELPER_NO_SCALING,
13775                                             DRM_PLANE_HELPER_NO_SCALING,
13776                                             true, true, &state->visible);
13777         if (ret)
13778                 return ret;
13779
13780         /* if we want to turn off the cursor ignore width and height */
13781         if (!obj)
13782                 return 0;
13783
13784         /* Check for which cursor types we support */
13785         if (!cursor_size_ok(plane->dev, state->base.crtc_w, state->base.crtc_h)) {
13786                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
13787                           state->base.crtc_w, state->base.crtc_h);
13788                 return -EINVAL;
13789         }
13790
13791         stride = roundup_pow_of_two(state->base.crtc_w) * 4;
13792         if (obj->base.size < stride * state->base.crtc_h) {
13793                 DRM_DEBUG_KMS("buffer is too small\n");
13794                 return -ENOMEM;
13795         }
13796
13797         if (fb->modifier[0] != DRM_FORMAT_MOD_NONE) {
13798                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
13799                 return -EINVAL;
13800         }
13801
13802         /*
13803          * There's something wrong with the cursor on CHV pipe C.
13804          * If it straddles the left edge of the screen then
13805          * moving it away from the edge or disabling it often
13806          * results in a pipe underrun, and often that can lead to
13807          * dead pipe (constant underrun reported, and it scans
13808          * out just a solid color). To recover from that, the
13809          * display power well must be turned off and on again.
13810          * Refuse the put the cursor into that compromised position.
13811          */
13812         if (IS_CHERRYVIEW(plane->dev) && pipe == PIPE_C &&
13813             state->visible && state->base.crtc_x < 0) {
13814                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
13815                 return -EINVAL;
13816         }
13817
13818         return 0;
13819 }
13820
13821 static void
13822 intel_disable_cursor_plane(struct drm_plane *plane,
13823                            struct drm_crtc *crtc)
13824 {
13825         intel_crtc_update_cursor(crtc, false);
13826 }
13827
13828 static void
13829 intel_commit_cursor_plane(struct drm_plane *plane,
13830                           struct intel_plane_state *state)
13831 {
13832         struct drm_crtc *crtc = state->base.crtc;
13833         struct drm_device *dev = plane->dev;
13834         struct intel_crtc *intel_crtc;
13835         struct drm_i915_gem_object *obj = intel_fb_obj(state->base.fb);
13836         uint32_t addr;
13837
13838         crtc = crtc ? crtc : plane->crtc;
13839         intel_crtc = to_intel_crtc(crtc);
13840
13841         if (!obj)
13842                 addr = 0;
13843         else if (!INTEL_INFO(dev)->cursor_needs_physical)
13844                 addr = i915_gem_obj_ggtt_offset(obj);
13845         else
13846                 addr = obj->phys_handle->busaddr;
13847
13848         intel_crtc->cursor_addr = addr;
13849
13850         if (crtc->state->active)
13851                 intel_crtc_update_cursor(crtc, state->visible);
13852 }
13853
13854 static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
13855                                                    int pipe)
13856 {
13857         struct intel_plane *cursor;
13858         struct intel_plane_state *state;
13859
13860         cursor = kzalloc(sizeof(*cursor), GFP_KERNEL);
13861         if (cursor == NULL)
13862                 return NULL;
13863
13864         state = intel_create_plane_state(&cursor->base);
13865         if (!state) {
13866                 kfree(cursor);
13867                 return NULL;
13868         }
13869         cursor->base.state = &state->base;
13870
13871         cursor->can_scale = false;
13872         cursor->max_downscale = 1;
13873         cursor->pipe = pipe;
13874         cursor->plane = pipe;
13875         cursor->frontbuffer_bit = INTEL_FRONTBUFFER_CURSOR(pipe);
13876         cursor->check_plane = intel_check_cursor_plane;
13877         cursor->commit_plane = intel_commit_cursor_plane;
13878         cursor->disable_plane = intel_disable_cursor_plane;
13879
13880         drm_universal_plane_init(dev, &cursor->base, 0,
13881                                  &intel_plane_funcs,
13882                                  intel_cursor_formats,
13883                                  ARRAY_SIZE(intel_cursor_formats),
13884                                  DRM_PLANE_TYPE_CURSOR);
13885
13886         if (INTEL_INFO(dev)->gen >= 4) {
13887                 if (!dev->mode_config.rotation_property)
13888                         dev->mode_config.rotation_property =
13889                                 drm_mode_create_rotation_property(dev,
13890                                                         BIT(DRM_ROTATE_0) |
13891                                                         BIT(DRM_ROTATE_180));
13892                 if (dev->mode_config.rotation_property)
13893                         drm_object_attach_property(&cursor->base.base,
13894                                 dev->mode_config.rotation_property,
13895                                 state->base.rotation);
13896         }
13897
13898         if (INTEL_INFO(dev)->gen >=9)
13899                 state->scaler_id = -1;
13900
13901         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
13902
13903         return &cursor->base;
13904 }
13905
13906 static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc,
13907         struct intel_crtc_state *crtc_state)
13908 {
13909         int i;
13910         struct intel_scaler *intel_scaler;
13911         struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state;
13912
13913         for (i = 0; i < intel_crtc->num_scalers; i++) {
13914                 intel_scaler = &scaler_state->scalers[i];
13915                 intel_scaler->in_use = 0;
13916                 intel_scaler->mode = PS_SCALER_MODE_DYN;
13917         }
13918
13919         scaler_state->scaler_id = -1;
13920 }
13921
13922 static void intel_crtc_init(struct drm_device *dev, int pipe)
13923 {
13924         struct drm_i915_private *dev_priv = dev->dev_private;
13925         struct intel_crtc *intel_crtc;
13926         struct intel_crtc_state *crtc_state = NULL;
13927         struct drm_plane *primary = NULL;
13928         struct drm_plane *cursor = NULL;
13929         int i, ret;
13930
13931         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
13932         if (intel_crtc == NULL)
13933                 return;
13934
13935         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
13936         if (!crtc_state)
13937                 goto fail;
13938         intel_crtc->config = crtc_state;
13939         intel_crtc->base.state = &crtc_state->base;
13940         crtc_state->base.crtc = &intel_crtc->base;
13941
13942         /* initialize shared scalers */
13943         if (INTEL_INFO(dev)->gen >= 9) {
13944                 if (pipe == PIPE_C)
13945                         intel_crtc->num_scalers = 1;
13946                 else
13947                         intel_crtc->num_scalers = SKL_NUM_SCALERS;
13948
13949                 skl_init_scalers(dev, intel_crtc, crtc_state);
13950         }
13951
13952         primary = intel_primary_plane_create(dev, pipe);
13953         if (!primary)
13954                 goto fail;
13955
13956         cursor = intel_cursor_plane_create(dev, pipe);
13957         if (!cursor)
13958                 goto fail;
13959
13960         ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary,
13961                                         cursor, &intel_crtc_funcs);
13962         if (ret)
13963                 goto fail;
13964
13965         drm_mode_crtc_set_gamma_size(&intel_crtc->base, 256);
13966         for (i = 0; i < 256; i++) {
13967                 intel_crtc->lut_r[i] = i;
13968                 intel_crtc->lut_g[i] = i;
13969                 intel_crtc->lut_b[i] = i;
13970         }
13971
13972         /*
13973          * On gen2/3 only plane A can do fbc, but the panel fitter and lvds port
13974          * is hooked to pipe B. Hence we want plane A feeding pipe B.
13975          */
13976         intel_crtc->pipe = pipe;
13977         intel_crtc->plane = pipe;
13978         if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4) {
13979                 DRM_DEBUG_KMS("swapping pipes & planes for FBC\n");
13980                 intel_crtc->plane = !pipe;
13981         }
13982
13983         intel_crtc->cursor_base = ~0;
13984         intel_crtc->cursor_cntl = ~0;
13985         intel_crtc->cursor_size = ~0;
13986
13987         intel_crtc->wm.cxsr_allowed = true;
13988
13989         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
13990                dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL);
13991         dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base;
13992         dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base;
13993
13994         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
13995
13996         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
13997         return;
13998
13999 fail:
14000         if (primary)
14001                 drm_plane_cleanup(primary);
14002         if (cursor)
14003                 drm_plane_cleanup(cursor);
14004         kfree(crtc_state);
14005         kfree(intel_crtc);
14006 }
14007
14008 enum pipe intel_get_pipe_from_connector(struct intel_connector *connector)
14009 {
14010         struct drm_encoder *encoder = connector->base.encoder;
14011         struct drm_device *dev = connector->base.dev;
14012
14013         WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
14014
14015         if (!encoder || WARN_ON(!encoder->crtc))
14016                 return INVALID_PIPE;
14017
14018         return to_intel_crtc(encoder->crtc)->pipe;
14019 }
14020
14021 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
14022                                 struct drm_file *file)
14023 {
14024         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
14025         struct drm_crtc *drmmode_crtc;
14026         struct intel_crtc *crtc;
14027
14028         drmmode_crtc = drm_crtc_find(dev, pipe_from_crtc_id->crtc_id);
14029
14030         if (!drmmode_crtc) {
14031                 DRM_ERROR("no such CRTC id\n");
14032                 return -ENOENT;
14033         }
14034
14035         crtc = to_intel_crtc(drmmode_crtc);
14036         pipe_from_crtc_id->pipe = crtc->pipe;
14037
14038         return 0;
14039 }
14040
14041 static int intel_encoder_clones(struct intel_encoder *encoder)
14042 {
14043         struct drm_device *dev = encoder->base.dev;
14044         struct intel_encoder *source_encoder;
14045         int index_mask = 0;
14046         int entry = 0;
14047
14048         for_each_intel_encoder(dev, source_encoder) {
14049                 if (encoders_cloneable(encoder, source_encoder))
14050                         index_mask |= (1 << entry);
14051
14052                 entry++;
14053         }
14054
14055         return index_mask;
14056 }
14057
14058 static bool has_edp_a(struct drm_device *dev)
14059 {
14060         struct drm_i915_private *dev_priv = dev->dev_private;
14061
14062         if (!IS_MOBILE(dev))
14063                 return false;
14064
14065         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
14066                 return false;
14067
14068         if (IS_GEN5(dev) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
14069                 return false;
14070
14071         return true;
14072 }
14073
14074 static bool intel_crt_present(struct drm_device *dev)
14075 {
14076         struct drm_i915_private *dev_priv = dev->dev_private;
14077
14078         if (INTEL_INFO(dev)->gen >= 9)
14079                 return false;
14080
14081         if (IS_HSW_ULT(dev) || IS_BDW_ULT(dev))
14082                 return false;
14083
14084         if (IS_CHERRYVIEW(dev))
14085                 return false;
14086
14087         if (IS_VALLEYVIEW(dev) && !dev_priv->vbt.int_crt_support)
14088                 return false;
14089
14090         return true;
14091 }
14092
14093 static void intel_setup_outputs(struct drm_device *dev)
14094 {
14095         struct drm_i915_private *dev_priv = dev->dev_private;
14096         struct intel_encoder *encoder;
14097         bool dpd_is_edp = false;
14098
14099         intel_lvds_init(dev);
14100
14101         if (intel_crt_present(dev))
14102                 intel_crt_init(dev);
14103
14104         if (IS_BROXTON(dev)) {
14105                 /*
14106                  * FIXME: Broxton doesn't support port detection via the
14107                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
14108                  * detect the ports.
14109                  */
14110                 intel_ddi_init(dev, PORT_A);
14111                 intel_ddi_init(dev, PORT_B);
14112                 intel_ddi_init(dev, PORT_C);
14113         } else if (HAS_DDI(dev)) {
14114                 int found;
14115
14116                 /*
14117                  * Haswell uses DDI functions to detect digital outputs.
14118                  * On SKL pre-D0 the strap isn't connected, so we assume
14119                  * it's there.
14120                  */
14121                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
14122                 /* WaIgnoreDDIAStrap: skl */
14123                 if (found || IS_SKYLAKE(dev))
14124                         intel_ddi_init(dev, PORT_A);
14125
14126                 /* DDI B, C and D detection is indicated by the SFUSE_STRAP
14127                  * register */
14128                 found = I915_READ(SFUSE_STRAP);
14129
14130                 if (found & SFUSE_STRAP_DDIB_DETECTED)
14131                         intel_ddi_init(dev, PORT_B);
14132                 if (found & SFUSE_STRAP_DDIC_DETECTED)
14133                         intel_ddi_init(dev, PORT_C);
14134                 if (found & SFUSE_STRAP_DDID_DETECTED)
14135                         intel_ddi_init(dev, PORT_D);
14136                 /*
14137                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
14138                  */
14139                 if (IS_SKYLAKE(dev) &&
14140                     (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp ||
14141                      dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi ||
14142                      dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi))
14143                         intel_ddi_init(dev, PORT_E);
14144
14145         } else if (HAS_PCH_SPLIT(dev)) {
14146                 int found;
14147                 dpd_is_edp = intel_dp_is_edp(dev, PORT_D);
14148
14149                 if (has_edp_a(dev))
14150                         intel_dp_init(dev, DP_A, PORT_A);
14151
14152                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
14153                         /* PCH SDVOB multiplex with HDMIB */
14154                         found = intel_sdvo_init(dev, PCH_SDVOB, true);
14155                         if (!found)
14156                                 intel_hdmi_init(dev, PCH_HDMIB, PORT_B);
14157                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
14158                                 intel_dp_init(dev, PCH_DP_B, PORT_B);
14159                 }
14160
14161                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
14162                         intel_hdmi_init(dev, PCH_HDMIC, PORT_C);
14163
14164                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
14165                         intel_hdmi_init(dev, PCH_HDMID, PORT_D);
14166
14167                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
14168                         intel_dp_init(dev, PCH_DP_C, PORT_C);
14169
14170                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
14171                         intel_dp_init(dev, PCH_DP_D, PORT_D);
14172         } else if (IS_VALLEYVIEW(dev)) {
14173                 /*
14174                  * The DP_DETECTED bit is the latched state of the DDC
14175                  * SDA pin at boot. However since eDP doesn't require DDC
14176                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
14177                  * eDP ports may have been muxed to an alternate function.
14178                  * Thus we can't rely on the DP_DETECTED bit alone to detect
14179                  * eDP ports. Consult the VBT as well as DP_DETECTED to
14180                  * detect eDP ports.
14181                  */
14182                 if (I915_READ(VLV_HDMIB) & SDVO_DETECTED &&
14183                     !intel_dp_is_edp(dev, PORT_B))
14184                         intel_hdmi_init(dev, VLV_HDMIB, PORT_B);
14185                 if (I915_READ(VLV_DP_B) & DP_DETECTED ||
14186                     intel_dp_is_edp(dev, PORT_B))
14187                         intel_dp_init(dev, VLV_DP_B, PORT_B);
14188
14189                 if (I915_READ(VLV_HDMIC) & SDVO_DETECTED &&
14190                     !intel_dp_is_edp(dev, PORT_C))
14191                         intel_hdmi_init(dev, VLV_HDMIC, PORT_C);
14192                 if (I915_READ(VLV_DP_C) & DP_DETECTED ||
14193                     intel_dp_is_edp(dev, PORT_C))
14194                         intel_dp_init(dev, VLV_DP_C, PORT_C);
14195
14196                 if (IS_CHERRYVIEW(dev)) {
14197                         /* eDP not supported on port D, so don't check VBT */
14198                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED)
14199                                 intel_hdmi_init(dev, CHV_HDMID, PORT_D);
14200                         if (I915_READ(CHV_DP_D) & DP_DETECTED)
14201                                 intel_dp_init(dev, CHV_DP_D, PORT_D);
14202                 }
14203
14204                 intel_dsi_init(dev);
14205         } else if (!IS_GEN2(dev) && !IS_PINEVIEW(dev)) {
14206                 bool found = false;
14207
14208                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14209                         DRM_DEBUG_KMS("probing SDVOB\n");
14210                         found = intel_sdvo_init(dev, GEN3_SDVOB, true);
14211                         if (!found && IS_G4X(dev)) {
14212                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
14213                                 intel_hdmi_init(dev, GEN4_HDMIB, PORT_B);
14214                         }
14215
14216                         if (!found && IS_G4X(dev))
14217                                 intel_dp_init(dev, DP_B, PORT_B);
14218                 }
14219
14220                 /* Before G4X SDVOC doesn't have its own detect register */
14221
14222                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14223                         DRM_DEBUG_KMS("probing SDVOC\n");
14224                         found = intel_sdvo_init(dev, GEN3_SDVOC, false);
14225                 }
14226
14227                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
14228
14229                         if (IS_G4X(dev)) {
14230                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
14231                                 intel_hdmi_init(dev, GEN4_HDMIC, PORT_C);
14232                         }
14233                         if (IS_G4X(dev))
14234                                 intel_dp_init(dev, DP_C, PORT_C);
14235                 }
14236
14237                 if (IS_G4X(dev) &&
14238                     (I915_READ(DP_D) & DP_DETECTED))
14239                         intel_dp_init(dev, DP_D, PORT_D);
14240         } else if (IS_GEN2(dev))
14241                 intel_dvo_init(dev);
14242
14243         if (SUPPORTS_TV(dev))
14244                 intel_tv_init(dev);
14245
14246         intel_psr_init(dev);
14247
14248         for_each_intel_encoder(dev, encoder) {
14249                 encoder->base.possible_crtcs = encoder->crtc_mask;
14250                 encoder->base.possible_clones =
14251                         intel_encoder_clones(encoder);
14252         }
14253
14254         intel_init_pch_refclk(dev);
14255
14256         drm_helper_move_panel_connectors_to_head(dev);
14257 }
14258
14259 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
14260 {
14261         struct drm_device *dev = fb->dev;
14262         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14263
14264         drm_framebuffer_cleanup(fb);
14265         mutex_lock(&dev->struct_mutex);
14266         WARN_ON(!intel_fb->obj->framebuffer_references--);
14267         drm_gem_object_unreference(&intel_fb->obj->base);
14268         mutex_unlock(&dev->struct_mutex);
14269         kfree(intel_fb);
14270 }
14271
14272 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
14273                                                 struct drm_file *file,
14274                                                 unsigned int *handle)
14275 {
14276         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14277         struct drm_i915_gem_object *obj = intel_fb->obj;
14278
14279         if (obj->userptr.mm) {
14280                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
14281                 return -EINVAL;
14282         }
14283
14284         return drm_gem_handle_create(file, &obj->base, handle);
14285 }
14286
14287 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
14288                                         struct drm_file *file,
14289                                         unsigned flags, unsigned color,
14290                                         struct drm_clip_rect *clips,
14291                                         unsigned num_clips)
14292 {
14293         struct drm_device *dev = fb->dev;
14294         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14295         struct drm_i915_gem_object *obj = intel_fb->obj;
14296
14297         mutex_lock(&dev->struct_mutex);
14298         intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB);
14299         mutex_unlock(&dev->struct_mutex);
14300
14301         return 0;
14302 }
14303
14304 static const struct drm_framebuffer_funcs intel_fb_funcs = {
14305         .destroy = intel_user_framebuffer_destroy,
14306         .create_handle = intel_user_framebuffer_create_handle,
14307         .dirty = intel_user_framebuffer_dirty,
14308 };
14309
14310 static
14311 u32 intel_fb_pitch_limit(struct drm_device *dev, uint64_t fb_modifier,
14312                          uint32_t pixel_format)
14313 {
14314         u32 gen = INTEL_INFO(dev)->gen;
14315
14316         if (gen >= 9) {
14317                 /* "The stride in bytes must not exceed the of the size of 8K
14318                  *  pixels and 32K bytes."
14319                  */
14320                  return min(8192*drm_format_plane_cpp(pixel_format, 0), 32768);
14321         } else if (gen >= 5 && !IS_VALLEYVIEW(dev)) {
14322                 return 32*1024;
14323         } else if (gen >= 4) {
14324                 if (fb_modifier == I915_FORMAT_MOD_X_TILED)
14325                         return 16*1024;
14326                 else
14327                         return 32*1024;
14328         } else if (gen >= 3) {
14329                 if (fb_modifier == I915_FORMAT_MOD_X_TILED)
14330                         return 8*1024;
14331                 else
14332                         return 16*1024;
14333         } else {
14334                 /* XXX DSPC is limited to 4k tiled */
14335                 return 8*1024;
14336         }
14337 }
14338
14339 static int intel_framebuffer_init(struct drm_device *dev,
14340                                   struct intel_framebuffer *intel_fb,
14341                                   struct drm_mode_fb_cmd2 *mode_cmd,
14342                                   struct drm_i915_gem_object *obj)
14343 {
14344         unsigned int aligned_height;
14345         int ret;
14346         u32 pitch_limit, stride_alignment;
14347
14348         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
14349
14350         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
14351                 /* Enforce that fb modifier and tiling mode match, but only for
14352                  * X-tiled. This is needed for FBC. */
14353                 if (!!(obj->tiling_mode == I915_TILING_X) !=
14354                     !!(mode_cmd->modifier[0] == I915_FORMAT_MOD_X_TILED)) {
14355                         DRM_DEBUG("tiling_mode doesn't match fb modifier\n");
14356                         return -EINVAL;
14357                 }
14358         } else {
14359                 if (obj->tiling_mode == I915_TILING_X)
14360                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
14361                 else if (obj->tiling_mode == I915_TILING_Y) {
14362                         DRM_DEBUG("No Y tiling for legacy addfb\n");
14363                         return -EINVAL;
14364                 }
14365         }
14366
14367         /* Passed in modifier sanity checking. */
14368         switch (mode_cmd->modifier[0]) {
14369         case I915_FORMAT_MOD_Y_TILED:
14370         case I915_FORMAT_MOD_Yf_TILED:
14371                 if (INTEL_INFO(dev)->gen < 9) {
14372                         DRM_DEBUG("Unsupported tiling 0x%llx!\n",
14373                                   mode_cmd->modifier[0]);
14374                         return -EINVAL;
14375                 }
14376         case DRM_FORMAT_MOD_NONE:
14377         case I915_FORMAT_MOD_X_TILED:
14378                 break;
14379         default:
14380                 DRM_DEBUG("Unsupported fb modifier 0x%llx!\n",
14381                           mode_cmd->modifier[0]);
14382                 return -EINVAL;
14383         }
14384
14385         stride_alignment = intel_fb_stride_alignment(dev, mode_cmd->modifier[0],
14386                                                      mode_cmd->pixel_format);
14387         if (mode_cmd->pitches[0] & (stride_alignment - 1)) {
14388                 DRM_DEBUG("pitch (%d) must be at least %u byte aligned\n",
14389                           mode_cmd->pitches[0], stride_alignment);
14390                 return -EINVAL;
14391         }
14392
14393         pitch_limit = intel_fb_pitch_limit(dev, mode_cmd->modifier[0],
14394                                            mode_cmd->pixel_format);
14395         if (mode_cmd->pitches[0] > pitch_limit) {
14396                 DRM_DEBUG("%s pitch (%u) must be at less than %d\n",
14397                           mode_cmd->modifier[0] != DRM_FORMAT_MOD_NONE ?
14398                           "tiled" : "linear",
14399                           mode_cmd->pitches[0], pitch_limit);
14400                 return -EINVAL;
14401         }
14402
14403         if (mode_cmd->modifier[0] == I915_FORMAT_MOD_X_TILED &&
14404             mode_cmd->pitches[0] != obj->stride) {
14405                 DRM_DEBUG("pitch (%d) must match tiling stride (%d)\n",
14406                           mode_cmd->pitches[0], obj->stride);
14407                 return -EINVAL;
14408         }
14409
14410         /* Reject formats not supported by any plane early. */
14411         switch (mode_cmd->pixel_format) {
14412         case DRM_FORMAT_C8:
14413         case DRM_FORMAT_RGB565:
14414         case DRM_FORMAT_XRGB8888:
14415         case DRM_FORMAT_ARGB8888:
14416                 break;
14417         case DRM_FORMAT_XRGB1555:
14418                 if (INTEL_INFO(dev)->gen > 3) {
14419                         DRM_DEBUG("unsupported pixel format: %s\n",
14420                                   drm_get_format_name(mode_cmd->pixel_format));
14421                         return -EINVAL;
14422                 }
14423                 break;
14424         case DRM_FORMAT_ABGR8888:
14425                 if (!IS_VALLEYVIEW(dev) && INTEL_INFO(dev)->gen < 9) {
14426                         DRM_DEBUG("unsupported pixel format: %s\n",
14427                                   drm_get_format_name(mode_cmd->pixel_format));
14428                         return -EINVAL;
14429                 }
14430                 break;
14431         case DRM_FORMAT_XBGR8888:
14432         case DRM_FORMAT_XRGB2101010:
14433         case DRM_FORMAT_XBGR2101010:
14434                 if (INTEL_INFO(dev)->gen < 4) {
14435                         DRM_DEBUG("unsupported pixel format: %s\n",
14436                                   drm_get_format_name(mode_cmd->pixel_format));
14437                         return -EINVAL;
14438                 }
14439                 break;
14440         case DRM_FORMAT_ABGR2101010:
14441                 if (!IS_VALLEYVIEW(dev)) {
14442                         DRM_DEBUG("unsupported pixel format: %s\n",
14443                                   drm_get_format_name(mode_cmd->pixel_format));
14444                         return -EINVAL;
14445                 }
14446                 break;
14447         case DRM_FORMAT_YUYV:
14448         case DRM_FORMAT_UYVY:
14449         case DRM_FORMAT_YVYU:
14450         case DRM_FORMAT_VYUY:
14451                 if (INTEL_INFO(dev)->gen < 5) {
14452                         DRM_DEBUG("unsupported pixel format: %s\n",
14453                                   drm_get_format_name(mode_cmd->pixel_format));
14454                         return -EINVAL;
14455                 }
14456                 break;
14457         default:
14458                 DRM_DEBUG("unsupported pixel format: %s\n",
14459                           drm_get_format_name(mode_cmd->pixel_format));
14460                 return -EINVAL;
14461         }
14462
14463         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
14464         if (mode_cmd->offsets[0] != 0)
14465                 return -EINVAL;
14466
14467         aligned_height = intel_fb_align_height(dev, mode_cmd->height,
14468                                                mode_cmd->pixel_format,
14469                                                mode_cmd->modifier[0]);
14470         /* FIXME drm helper for size checks (especially planar formats)? */
14471         if (obj->base.size < aligned_height * mode_cmd->pitches[0])
14472                 return -EINVAL;
14473
14474         drm_helper_mode_fill_fb_struct(&intel_fb->base, mode_cmd);
14475         intel_fb->obj = obj;
14476         intel_fb->obj->framebuffer_references++;
14477
14478         ret = drm_framebuffer_init(dev, &intel_fb->base, &intel_fb_funcs);
14479         if (ret) {
14480                 DRM_ERROR("framebuffer init failed %d\n", ret);
14481                 return ret;
14482         }
14483
14484         return 0;
14485 }
14486
14487 static struct drm_framebuffer *
14488 intel_user_framebuffer_create(struct drm_device *dev,
14489                               struct drm_file *filp,
14490                               struct drm_mode_fb_cmd2 *user_mode_cmd)
14491 {
14492         struct drm_i915_gem_object *obj;
14493         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
14494
14495         obj = to_intel_bo(drm_gem_object_lookup(dev, filp,
14496                                                 mode_cmd.handles[0]));
14497         if (&obj->base == NULL)
14498                 return ERR_PTR(-ENOENT);
14499
14500         return intel_framebuffer_create(dev, &mode_cmd, obj);
14501 }
14502
14503 #ifndef CONFIG_DRM_FBDEV_EMULATION
14504 static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
14505 {
14506 }
14507 #endif
14508
14509 static const struct drm_mode_config_funcs intel_mode_funcs = {
14510         .fb_create = intel_user_framebuffer_create,
14511         .output_poll_changed = intel_fbdev_output_poll_changed,
14512         .atomic_check = intel_atomic_check,
14513         .atomic_commit = intel_atomic_commit,
14514         .atomic_state_alloc = intel_atomic_state_alloc,
14515         .atomic_state_clear = intel_atomic_state_clear,
14516 };
14517
14518 /* Set up chip specific display functions */
14519 static void intel_init_display(struct drm_device *dev)
14520 {
14521         struct drm_i915_private *dev_priv = dev->dev_private;
14522
14523         if (HAS_PCH_SPLIT(dev) || IS_G4X(dev))
14524                 dev_priv->display.find_dpll = g4x_find_best_dpll;
14525         else if (IS_CHERRYVIEW(dev))
14526                 dev_priv->display.find_dpll = chv_find_best_dpll;
14527         else if (IS_VALLEYVIEW(dev))
14528                 dev_priv->display.find_dpll = vlv_find_best_dpll;
14529         else if (IS_PINEVIEW(dev))
14530                 dev_priv->display.find_dpll = pnv_find_best_dpll;
14531         else
14532                 dev_priv->display.find_dpll = i9xx_find_best_dpll;
14533
14534         if (INTEL_INFO(dev)->gen >= 9) {
14535                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14536                 dev_priv->display.get_initial_plane_config =
14537                         skylake_get_initial_plane_config;
14538                 dev_priv->display.crtc_compute_clock =
14539                         haswell_crtc_compute_clock;
14540                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14541                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14542                 dev_priv->display.update_primary_plane =
14543                         skylake_update_primary_plane;
14544         } else if (HAS_DDI(dev)) {
14545                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14546                 dev_priv->display.get_initial_plane_config =
14547                         ironlake_get_initial_plane_config;
14548                 dev_priv->display.crtc_compute_clock =
14549                         haswell_crtc_compute_clock;
14550                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14551                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14552                 dev_priv->display.update_primary_plane =
14553                         ironlake_update_primary_plane;
14554         } else if (HAS_PCH_SPLIT(dev)) {
14555                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
14556                 dev_priv->display.get_initial_plane_config =
14557                         ironlake_get_initial_plane_config;
14558                 dev_priv->display.crtc_compute_clock =
14559                         ironlake_crtc_compute_clock;
14560                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
14561                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
14562                 dev_priv->display.update_primary_plane =
14563                         ironlake_update_primary_plane;
14564         } else if (IS_VALLEYVIEW(dev)) {
14565                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14566                 dev_priv->display.get_initial_plane_config =
14567                         i9xx_get_initial_plane_config;
14568                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
14569                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14570                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14571                 dev_priv->display.update_primary_plane =
14572                         i9xx_update_primary_plane;
14573         } else {
14574                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14575                 dev_priv->display.get_initial_plane_config =
14576                         i9xx_get_initial_plane_config;
14577                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
14578                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14579                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14580                 dev_priv->display.update_primary_plane =
14581                         i9xx_update_primary_plane;
14582         }
14583
14584         /* Returns the core display clock speed */
14585         if (IS_SKYLAKE(dev))
14586                 dev_priv->display.get_display_clock_speed =
14587                         skylake_get_display_clock_speed;
14588         else if (IS_BROXTON(dev))
14589                 dev_priv->display.get_display_clock_speed =
14590                         broxton_get_display_clock_speed;
14591         else if (IS_BROADWELL(dev))
14592                 dev_priv->display.get_display_clock_speed =
14593                         broadwell_get_display_clock_speed;
14594         else if (IS_HASWELL(dev))
14595                 dev_priv->display.get_display_clock_speed =
14596                         haswell_get_display_clock_speed;
14597         else if (IS_VALLEYVIEW(dev))
14598                 dev_priv->display.get_display_clock_speed =
14599                         valleyview_get_display_clock_speed;
14600         else if (IS_GEN5(dev))
14601                 dev_priv->display.get_display_clock_speed =
14602                         ilk_get_display_clock_speed;
14603         else if (IS_I945G(dev) || IS_BROADWATER(dev) ||
14604                  IS_GEN6(dev) || IS_IVYBRIDGE(dev))
14605                 dev_priv->display.get_display_clock_speed =
14606                         i945_get_display_clock_speed;
14607         else if (IS_GM45(dev))
14608                 dev_priv->display.get_display_clock_speed =
14609                         gm45_get_display_clock_speed;
14610         else if (IS_CRESTLINE(dev))
14611                 dev_priv->display.get_display_clock_speed =
14612                         i965gm_get_display_clock_speed;
14613         else if (IS_PINEVIEW(dev))
14614                 dev_priv->display.get_display_clock_speed =
14615                         pnv_get_display_clock_speed;
14616         else if (IS_G33(dev) || IS_G4X(dev))
14617                 dev_priv->display.get_display_clock_speed =
14618                         g33_get_display_clock_speed;
14619         else if (IS_I915G(dev))
14620                 dev_priv->display.get_display_clock_speed =
14621                         i915_get_display_clock_speed;
14622         else if (IS_I945GM(dev) || IS_845G(dev))
14623                 dev_priv->display.get_display_clock_speed =
14624                         i9xx_misc_get_display_clock_speed;
14625         else if (IS_PINEVIEW(dev))
14626                 dev_priv->display.get_display_clock_speed =
14627                         pnv_get_display_clock_speed;
14628         else if (IS_I915GM(dev))
14629                 dev_priv->display.get_display_clock_speed =
14630                         i915gm_get_display_clock_speed;
14631         else if (IS_I865G(dev))
14632                 dev_priv->display.get_display_clock_speed =
14633                         i865_get_display_clock_speed;
14634         else if (IS_I85X(dev))
14635                 dev_priv->display.get_display_clock_speed =
14636                         i85x_get_display_clock_speed;
14637         else { /* 830 */
14638                 WARN(!IS_I830(dev), "Unknown platform. Assuming 133 MHz CDCLK\n");
14639                 dev_priv->display.get_display_clock_speed =
14640                         i830_get_display_clock_speed;
14641         }
14642
14643         if (IS_GEN5(dev)) {
14644                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
14645         } else if (IS_GEN6(dev)) {
14646                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
14647         } else if (IS_IVYBRIDGE(dev)) {
14648                 /* FIXME: detect B0+ stepping and use auto training */
14649                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
14650         } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
14651                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
14652                 if (IS_BROADWELL(dev)) {
14653                         dev_priv->display.modeset_commit_cdclk =
14654                                 broadwell_modeset_commit_cdclk;
14655                         dev_priv->display.modeset_calc_cdclk =
14656                                 broadwell_modeset_calc_cdclk;
14657                 }
14658         } else if (IS_VALLEYVIEW(dev)) {
14659                 dev_priv->display.modeset_commit_cdclk =
14660                         valleyview_modeset_commit_cdclk;
14661                 dev_priv->display.modeset_calc_cdclk =
14662                         valleyview_modeset_calc_cdclk;
14663         } else if (IS_BROXTON(dev)) {
14664                 dev_priv->display.modeset_commit_cdclk =
14665                         broxton_modeset_commit_cdclk;
14666                 dev_priv->display.modeset_calc_cdclk =
14667                         broxton_modeset_calc_cdclk;
14668         }
14669
14670         switch (INTEL_INFO(dev)->gen) {
14671         case 2:
14672                 dev_priv->display.queue_flip = intel_gen2_queue_flip;
14673                 break;
14674
14675         case 3:
14676                 dev_priv->display.queue_flip = intel_gen3_queue_flip;
14677                 break;
14678
14679         case 4:
14680         case 5:
14681                 dev_priv->display.queue_flip = intel_gen4_queue_flip;
14682                 break;
14683
14684         case 6:
14685                 dev_priv->display.queue_flip = intel_gen6_queue_flip;
14686                 break;
14687         case 7:
14688         case 8: /* FIXME(BDW): Check that the gen8 RCS flip works. */
14689                 dev_priv->display.queue_flip = intel_gen7_queue_flip;
14690                 break;
14691         case 9:
14692                 /* Drop through - unsupported since execlist only. */
14693         default:
14694                 /* Default just returns -ENODEV to indicate unsupported */
14695                 dev_priv->display.queue_flip = intel_default_queue_flip;
14696         }
14697
14698         mutex_init(&dev_priv->pps_mutex);
14699 }
14700
14701 /*
14702  * Some BIOSes insist on assuming the GPU's pipe A is enabled at suspend,
14703  * resume, or other times.  This quirk makes sure that's the case for
14704  * affected systems.
14705  */
14706 static void quirk_pipea_force(struct drm_device *dev)
14707 {
14708         struct drm_i915_private *dev_priv = dev->dev_private;
14709
14710         dev_priv->quirks |= QUIRK_PIPEA_FORCE;
14711         DRM_INFO("applying pipe a force quirk\n");
14712 }
14713
14714 static void quirk_pipeb_force(struct drm_device *dev)
14715 {
14716         struct drm_i915_private *dev_priv = dev->dev_private;
14717
14718         dev_priv->quirks |= QUIRK_PIPEB_FORCE;
14719         DRM_INFO("applying pipe b force quirk\n");
14720 }
14721
14722 /*
14723  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
14724  */
14725 static void quirk_ssc_force_disable(struct drm_device *dev)
14726 {
14727         struct drm_i915_private *dev_priv = dev->dev_private;
14728         dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
14729         DRM_INFO("applying lvds SSC disable quirk\n");
14730 }
14731
14732 /*
14733  * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight
14734  * brightness value
14735  */
14736 static void quirk_invert_brightness(struct drm_device *dev)
14737 {
14738         struct drm_i915_private *dev_priv = dev->dev_private;
14739         dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS;
14740         DRM_INFO("applying inverted panel brightness quirk\n");
14741 }
14742
14743 /* Some VBT's incorrectly indicate no backlight is present */
14744 static void quirk_backlight_present(struct drm_device *dev)
14745 {
14746         struct drm_i915_private *dev_priv = dev->dev_private;
14747         dev_priv->quirks |= QUIRK_BACKLIGHT_PRESENT;
14748         DRM_INFO("applying backlight present quirk\n");
14749 }
14750
14751 struct intel_quirk {
14752         int device;
14753         int subsystem_vendor;
14754         int subsystem_device;
14755         void (*hook)(struct drm_device *dev);
14756 };
14757
14758 /* For systems that don't have a meaningful PCI subdevice/subvendor ID */
14759 struct intel_dmi_quirk {
14760         void (*hook)(struct drm_device *dev);
14761         const struct dmi_system_id (*dmi_id_list)[];
14762 };
14763
14764 static int intel_dmi_reverse_brightness(const struct dmi_system_id *id)
14765 {
14766         DRM_INFO("Backlight polarity reversed on %s\n", id->ident);
14767         return 1;
14768 }
14769
14770 static const struct intel_dmi_quirk intel_dmi_quirks[] = {
14771         {
14772                 .dmi_id_list = &(const struct dmi_system_id[]) {
14773                         {
14774                                 .callback = intel_dmi_reverse_brightness,
14775                                 .ident = "NCR Corporation",
14776                                 .matches = {DMI_MATCH(DMI_SYS_VENDOR, "NCR Corporation"),
14777                                             DMI_MATCH(DMI_PRODUCT_NAME, ""),
14778                                 },
14779                         },
14780                         { }  /* terminating entry */
14781                 },
14782                 .hook = quirk_invert_brightness,
14783         },
14784 };
14785
14786 static struct intel_quirk intel_quirks[] = {
14787         /* Toshiba Protege R-205, S-209 needs pipe A force quirk */
14788         { 0x2592, 0x1179, 0x0001, quirk_pipea_force },
14789
14790         /* ThinkPad T60 needs pipe A force quirk (bug #16494) */
14791         { 0x2782, 0x17aa, 0x201a, quirk_pipea_force },
14792
14793         /* 830 needs to leave pipe A & dpll A up */
14794         { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
14795
14796         /* 830 needs to leave pipe B & dpll B up */
14797         { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipeb_force },
14798
14799         /* Lenovo U160 cannot use SSC on LVDS */
14800         { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
14801
14802         /* Sony Vaio Y cannot use SSC on LVDS */
14803         { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable },
14804
14805         /* Acer Aspire 5734Z must invert backlight brightness */
14806         { 0x2a42, 0x1025, 0x0459, quirk_invert_brightness },
14807
14808         /* Acer/eMachines G725 */
14809         { 0x2a42, 0x1025, 0x0210, quirk_invert_brightness },
14810
14811         /* Acer/eMachines e725 */
14812         { 0x2a42, 0x1025, 0x0212, quirk_invert_brightness },
14813
14814         /* Acer/Packard Bell NCL20 */
14815         { 0x2a42, 0x1025, 0x034b, quirk_invert_brightness },
14816
14817         /* Acer Aspire 4736Z */
14818         { 0x2a42, 0x1025, 0x0260, quirk_invert_brightness },
14819
14820         /* Acer Aspire 5336 */
14821         { 0x2a42, 0x1025, 0x048a, quirk_invert_brightness },
14822
14823         /* Acer C720 and C720P Chromebooks (Celeron 2955U) have backlights */
14824         { 0x0a06, 0x1025, 0x0a11, quirk_backlight_present },
14825
14826         /* Acer C720 Chromebook (Core i3 4005U) */
14827         { 0x0a16, 0x1025, 0x0a11, quirk_backlight_present },
14828
14829         /* Apple Macbook 2,1 (Core 2 T7400) */
14830         { 0x27a2, 0x8086, 0x7270, quirk_backlight_present },
14831
14832         /* Apple Macbook 4,1 */
14833         { 0x2a02, 0x106b, 0x00a1, quirk_backlight_present },
14834
14835         /* Toshiba CB35 Chromebook (Celeron 2955U) */
14836         { 0x0a06, 0x1179, 0x0a88, quirk_backlight_present },
14837
14838         /* HP Chromebook 14 (Celeron 2955U) */
14839         { 0x0a06, 0x103c, 0x21ed, quirk_backlight_present },
14840
14841         /* Dell Chromebook 11 */
14842         { 0x0a06, 0x1028, 0x0a35, quirk_backlight_present },
14843
14844         /* Dell Chromebook 11 (2015 version) */
14845         { 0x0a16, 0x1028, 0x0a35, quirk_backlight_present },
14846 };
14847
14848 static void intel_init_quirks(struct drm_device *dev)
14849 {
14850         struct pci_dev *d = dev->pdev;
14851         int i;
14852
14853         for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) {
14854                 struct intel_quirk *q = &intel_quirks[i];
14855
14856                 if (d->device == q->device &&
14857                     (d->subsystem_vendor == q->subsystem_vendor ||
14858                      q->subsystem_vendor == PCI_ANY_ID) &&
14859                     (d->subsystem_device == q->subsystem_device ||
14860                      q->subsystem_device == PCI_ANY_ID))
14861                         q->hook(dev);
14862         }
14863         for (i = 0; i < ARRAY_SIZE(intel_dmi_quirks); i++) {
14864                 if (dmi_check_system(*intel_dmi_quirks[i].dmi_id_list) != 0)
14865                         intel_dmi_quirks[i].hook(dev);
14866         }
14867 }
14868
14869 /* Disable the VGA plane that we never use */
14870 static void i915_disable_vga(struct drm_device *dev)
14871 {
14872         struct drm_i915_private *dev_priv = dev->dev_private;
14873         u8 sr1;
14874         u32 vga_reg = i915_vgacntrl_reg(dev);
14875
14876         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
14877         vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
14878         outb(SR01, VGA_SR_INDEX);
14879         sr1 = inb(VGA_SR_DATA);
14880         outb(sr1 | 1<<5, VGA_SR_DATA);
14881         vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
14882         udelay(300);
14883
14884         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
14885         POSTING_READ(vga_reg);
14886 }
14887
14888 void intel_modeset_init_hw(struct drm_device *dev)
14889 {
14890         intel_update_cdclk(dev);
14891         intel_prepare_ddi(dev);
14892         intel_init_clock_gating(dev);
14893         intel_enable_gt_powersave(dev);
14894 }
14895
14896 void intel_modeset_init(struct drm_device *dev)
14897 {
14898         struct drm_i915_private *dev_priv = dev->dev_private;
14899         int sprite, ret;
14900         enum pipe pipe;
14901         struct intel_crtc *crtc;
14902
14903         drm_mode_config_init(dev);
14904
14905         dev->mode_config.min_width = 0;
14906         dev->mode_config.min_height = 0;
14907
14908         dev->mode_config.preferred_depth = 24;
14909         dev->mode_config.prefer_shadow = 1;
14910
14911         dev->mode_config.allow_fb_modifiers = true;
14912
14913         dev->mode_config.funcs = &intel_mode_funcs;
14914
14915         intel_init_quirks(dev);
14916
14917         intel_init_pm(dev);
14918
14919         if (INTEL_INFO(dev)->num_pipes == 0)
14920                 return;
14921
14922         /*
14923          * There may be no VBT; and if the BIOS enabled SSC we can
14924          * just keep using it to avoid unnecessary flicker.  Whereas if the
14925          * BIOS isn't using it, don't assume it will work even if the VBT
14926          * indicates as much.
14927          */
14928         if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
14929                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
14930                                             DREF_SSC1_ENABLE);
14931
14932                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
14933                         DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n",
14934                                      bios_lvds_use_ssc ? "en" : "dis",
14935                                      dev_priv->vbt.lvds_use_ssc ? "en" : "dis");
14936                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
14937                 }
14938         }
14939
14940         intel_init_display(dev);
14941         intel_init_audio(dev);
14942
14943         if (IS_GEN2(dev)) {
14944                 dev->mode_config.max_width = 2048;
14945                 dev->mode_config.max_height = 2048;
14946         } else if (IS_GEN3(dev)) {
14947                 dev->mode_config.max_width = 4096;
14948                 dev->mode_config.max_height = 4096;
14949         } else {
14950                 dev->mode_config.max_width = 8192;
14951                 dev->mode_config.max_height = 8192;
14952         }
14953
14954         if (IS_845G(dev) || IS_I865G(dev)) {
14955                 dev->mode_config.cursor_width = IS_845G(dev) ? 64 : 512;
14956                 dev->mode_config.cursor_height = 1023;
14957         } else if (IS_GEN2(dev)) {
14958                 dev->mode_config.cursor_width = GEN2_CURSOR_WIDTH;
14959                 dev->mode_config.cursor_height = GEN2_CURSOR_HEIGHT;
14960         } else {
14961                 dev->mode_config.cursor_width = MAX_CURSOR_WIDTH;
14962                 dev->mode_config.cursor_height = MAX_CURSOR_HEIGHT;
14963         }
14964
14965         dev->mode_config.fb_base = dev_priv->gtt.mappable_base;
14966
14967         DRM_DEBUG_KMS("%d display pipe%s available.\n",
14968                       INTEL_INFO(dev)->num_pipes,
14969                       INTEL_INFO(dev)->num_pipes > 1 ? "s" : "");
14970
14971         for_each_pipe(dev_priv, pipe) {
14972                 intel_crtc_init(dev, pipe);
14973                 for_each_sprite(dev_priv, pipe, sprite) {
14974                         ret = intel_plane_init(dev, pipe, sprite);
14975                         if (ret)
14976                                 DRM_DEBUG_KMS("pipe %c sprite %c init failed: %d\n",
14977                                               pipe_name(pipe), sprite_name(pipe, sprite), ret);
14978                 }
14979         }
14980
14981         intel_update_czclk(dev_priv);
14982         intel_update_cdclk(dev);
14983
14984         intel_shared_dpll_init(dev);
14985
14986         /* Just disable it once at startup */
14987         i915_disable_vga(dev);
14988         intel_setup_outputs(dev);
14989
14990         /* Just in case the BIOS is doing something questionable. */
14991         intel_fbc_disable(dev_priv);
14992
14993         drm_modeset_lock_all(dev);
14994         intel_modeset_setup_hw_state(dev);
14995         drm_modeset_unlock_all(dev);
14996
14997         for_each_intel_crtc(dev, crtc) {
14998                 struct intel_initial_plane_config plane_config = {};
14999
15000                 if (!crtc->active)
15001                         continue;
15002
15003                 /*
15004                  * Note that reserving the BIOS fb up front prevents us
15005                  * from stuffing other stolen allocations like the ring
15006                  * on top.  This prevents some ugliness at boot time, and
15007                  * can even allow for smooth boot transitions if the BIOS
15008                  * fb is large enough for the active pipe configuration.
15009                  */
15010                 dev_priv->display.get_initial_plane_config(crtc,
15011                                                            &plane_config);
15012
15013                 /*
15014                  * If the fb is shared between multiple heads, we'll
15015                  * just get the first one.
15016                  */
15017                 intel_find_initial_plane_obj(crtc, &plane_config);
15018         }
15019 }
15020
15021 static void intel_enable_pipe_a(struct drm_device *dev)
15022 {
15023         struct intel_connector *connector;
15024         struct drm_connector *crt = NULL;
15025         struct intel_load_detect_pipe load_detect_temp;
15026         struct drm_modeset_acquire_ctx *ctx = dev->mode_config.acquire_ctx;
15027
15028         /* We can't just switch on the pipe A, we need to set things up with a
15029          * proper mode and output configuration. As a gross hack, enable pipe A
15030          * by enabling the load detect pipe once. */
15031         for_each_intel_connector(dev, connector) {
15032                 if (connector->encoder->type == INTEL_OUTPUT_ANALOG) {
15033                         crt = &connector->base;
15034                         break;
15035                 }
15036         }
15037
15038         if (!crt)
15039                 return;
15040
15041         if (intel_get_load_detect_pipe(crt, NULL, &load_detect_temp, ctx))
15042                 intel_release_load_detect_pipe(crt, &load_detect_temp, ctx);
15043 }
15044
15045 static bool
15046 intel_check_plane_mapping(struct intel_crtc *crtc)
15047 {
15048         struct drm_device *dev = crtc->base.dev;
15049         struct drm_i915_private *dev_priv = dev->dev_private;
15050         u32 val;
15051
15052         if (INTEL_INFO(dev)->num_pipes == 1)
15053                 return true;
15054
15055         val = I915_READ(DSPCNTR(!crtc->plane));
15056
15057         if ((val & DISPLAY_PLANE_ENABLE) &&
15058             (!!(val & DISPPLANE_SEL_PIPE_MASK) == crtc->pipe))
15059                 return false;
15060
15061         return true;
15062 }
15063
15064 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
15065 {
15066         struct drm_device *dev = crtc->base.dev;
15067         struct intel_encoder *encoder;
15068
15069         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
15070                 return true;
15071
15072         return false;
15073 }
15074
15075 static void intel_sanitize_crtc(struct intel_crtc *crtc)
15076 {
15077         struct drm_device *dev = crtc->base.dev;
15078         struct drm_i915_private *dev_priv = dev->dev_private;
15079         u32 reg;
15080
15081         /* Clear any frame start delays used for debugging left by the BIOS */
15082         reg = PIPECONF(crtc->config->cpu_transcoder);
15083         I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
15084
15085         /* restore vblank interrupts to correct state */
15086         drm_crtc_vblank_reset(&crtc->base);
15087         if (crtc->active) {
15088                 struct intel_plane *plane;
15089
15090                 drm_crtc_vblank_on(&crtc->base);
15091
15092                 /* Disable everything but the primary plane */
15093                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
15094                         if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
15095                                 continue;
15096
15097                         plane->disable_plane(&plane->base, &crtc->base);
15098                 }
15099         }
15100
15101         /* We need to sanitize the plane -> pipe mapping first because this will
15102          * disable the crtc (and hence change the state) if it is wrong. Note
15103          * that gen4+ has a fixed plane -> pipe mapping.  */
15104         if (INTEL_INFO(dev)->gen < 4 && !intel_check_plane_mapping(crtc)) {
15105                 bool plane;
15106
15107                 DRM_DEBUG_KMS("[CRTC:%d] wrong plane connection detected!\n",
15108                               crtc->base.base.id);
15109
15110                 /* Pipe has the wrong plane attached and the plane is active.
15111                  * Temporarily change the plane mapping and disable everything
15112                  * ...  */
15113                 plane = crtc->plane;
15114                 to_intel_plane_state(crtc->base.primary->state)->visible = true;
15115                 crtc->plane = !plane;
15116                 intel_crtc_disable_noatomic(&crtc->base);
15117                 crtc->plane = plane;
15118         }
15119
15120         if (dev_priv->quirks & QUIRK_PIPEA_FORCE &&
15121             crtc->pipe == PIPE_A && !crtc->active) {
15122                 /* BIOS forgot to enable pipe A, this mostly happens after
15123                  * resume. Force-enable the pipe to fix this, the update_dpms
15124                  * call below we restore the pipe to the right state, but leave
15125                  * the required bits on. */
15126                 intel_enable_pipe_a(dev);
15127         }
15128
15129         /* Adjust the state of the output pipe according to whether we
15130          * have active connectors/encoders. */
15131         if (!intel_crtc_has_encoders(crtc))
15132                 intel_crtc_disable_noatomic(&crtc->base);
15133
15134         if (crtc->active != crtc->base.state->active) {
15135                 struct intel_encoder *encoder;
15136
15137                 /* This can happen either due to bugs in the get_hw_state
15138                  * functions or because of calls to intel_crtc_disable_noatomic,
15139                  * or because the pipe is force-enabled due to the
15140                  * pipe A quirk. */
15141                 DRM_DEBUG_KMS("[CRTC:%d] hw state adjusted, was %s, now %s\n",
15142                               crtc->base.base.id,
15143                               crtc->base.state->enable ? "enabled" : "disabled",
15144                               crtc->active ? "enabled" : "disabled");
15145
15146                 WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, NULL) < 0);
15147                 crtc->base.state->active = crtc->active;
15148                 crtc->base.enabled = crtc->active;
15149
15150                 /* Because we only establish the connector -> encoder ->
15151                  * crtc links if something is active, this means the
15152                  * crtc is now deactivated. Break the links. connector
15153                  * -> encoder links are only establish when things are
15154                  *  actually up, hence no need to break them. */
15155                 WARN_ON(crtc->active);
15156
15157                 for_each_encoder_on_crtc(dev, &crtc->base, encoder)
15158                         encoder->base.crtc = NULL;
15159         }
15160
15161         if (crtc->active || HAS_GMCH_DISPLAY(dev)) {
15162                 /*
15163                  * We start out with underrun reporting disabled to avoid races.
15164                  * For correct bookkeeping mark this on active crtcs.
15165                  *
15166                  * Also on gmch platforms we dont have any hardware bits to
15167                  * disable the underrun reporting. Which means we need to start
15168                  * out with underrun reporting disabled also on inactive pipes,
15169                  * since otherwise we'll complain about the garbage we read when
15170                  * e.g. coming up after runtime pm.
15171                  *
15172                  * No protection against concurrent access is required - at
15173                  * worst a fifo underrun happens which also sets this to false.
15174                  */
15175                 crtc->cpu_fifo_underrun_disabled = true;
15176                 crtc->pch_fifo_underrun_disabled = true;
15177         }
15178 }
15179
15180 static void intel_sanitize_encoder(struct intel_encoder *encoder)
15181 {
15182         struct intel_connector *connector;
15183         struct drm_device *dev = encoder->base.dev;
15184         bool active = false;
15185
15186         /* We need to check both for a crtc link (meaning that the
15187          * encoder is active and trying to read from a pipe) and the
15188          * pipe itself being active. */
15189         bool has_active_crtc = encoder->base.crtc &&
15190                 to_intel_crtc(encoder->base.crtc)->active;
15191
15192         for_each_intel_connector(dev, connector) {
15193                 if (connector->base.encoder != &encoder->base)
15194                         continue;
15195
15196                 active = true;
15197                 break;
15198         }
15199
15200         if (active && !has_active_crtc) {
15201                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
15202                               encoder->base.base.id,
15203                               encoder->base.name);
15204
15205                 /* Connector is active, but has no active pipe. This is
15206                  * fallout from our resume register restoring. Disable
15207                  * the encoder manually again. */
15208                 if (encoder->base.crtc) {
15209                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
15210                                       encoder->base.base.id,
15211                                       encoder->base.name);
15212                         encoder->disable(encoder);
15213                         if (encoder->post_disable)
15214                                 encoder->post_disable(encoder);
15215                 }
15216                 encoder->base.crtc = NULL;
15217
15218                 /* Inconsistent output/port/pipe state happens presumably due to
15219                  * a bug in one of the get_hw_state functions. Or someplace else
15220                  * in our code, like the register restore mess on resume. Clamp
15221                  * things to off as a safer default. */
15222                 for_each_intel_connector(dev, connector) {
15223                         if (connector->encoder != encoder)
15224                                 continue;
15225                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15226                         connector->base.encoder = NULL;
15227                 }
15228         }
15229         /* Enabled encoders without active connectors will be fixed in
15230          * the crtc fixup. */
15231 }
15232
15233 void i915_redisable_vga_power_on(struct drm_device *dev)
15234 {
15235         struct drm_i915_private *dev_priv = dev->dev_private;
15236         u32 vga_reg = i915_vgacntrl_reg(dev);
15237
15238         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
15239                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
15240                 i915_disable_vga(dev);
15241         }
15242 }
15243
15244 void i915_redisable_vga(struct drm_device *dev)
15245 {
15246         struct drm_i915_private *dev_priv = dev->dev_private;
15247
15248         /* This function can be called both from intel_modeset_setup_hw_state or
15249          * at a very early point in our resume sequence, where the power well
15250          * structures are not yet restored. Since this function is at a very
15251          * paranoid "someone might have enabled VGA while we were not looking"
15252          * level, just check if the power well is enabled instead of trying to
15253          * follow the "don't touch the power well if we don't need it" policy
15254          * the rest of the driver uses. */
15255         if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_VGA))
15256                 return;
15257
15258         i915_redisable_vga_power_on(dev);
15259 }
15260
15261 static bool primary_get_hw_state(struct intel_plane *plane)
15262 {
15263         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
15264
15265         return I915_READ(DSPCNTR(plane->plane)) & DISPLAY_PLANE_ENABLE;
15266 }
15267
15268 /* FIXME read out full plane state for all planes */
15269 static void readout_plane_state(struct intel_crtc *crtc)
15270 {
15271         struct drm_plane *primary = crtc->base.primary;
15272         struct intel_plane_state *plane_state =
15273                 to_intel_plane_state(primary->state);
15274
15275         plane_state->visible =
15276                 primary_get_hw_state(to_intel_plane(primary));
15277
15278         if (plane_state->visible)
15279                 crtc->base.state->plane_mask |= 1 << drm_plane_index(primary);
15280 }
15281
15282 static void intel_modeset_readout_hw_state(struct drm_device *dev)
15283 {
15284         struct drm_i915_private *dev_priv = dev->dev_private;
15285         enum pipe pipe;
15286         struct intel_crtc *crtc;
15287         struct intel_encoder *encoder;
15288         struct intel_connector *connector;
15289         int i;
15290
15291         for_each_intel_crtc(dev, crtc) {
15292                 __drm_atomic_helper_crtc_destroy_state(&crtc->base, crtc->base.state);
15293                 memset(crtc->config, 0, sizeof(*crtc->config));
15294                 crtc->config->base.crtc = &crtc->base;
15295
15296                 crtc->active = dev_priv->display.get_pipe_config(crtc,
15297                                                                  crtc->config);
15298
15299                 crtc->base.state->active = crtc->active;
15300                 crtc->base.enabled = crtc->active;
15301
15302                 readout_plane_state(crtc);
15303
15304                 DRM_DEBUG_KMS("[CRTC:%d] hw state readout: %s\n",
15305                               crtc->base.base.id,
15306                               crtc->active ? "enabled" : "disabled");
15307         }
15308
15309         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15310                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15311
15312                 pll->on = pll->get_hw_state(dev_priv, pll,
15313                                             &pll->config.hw_state);
15314                 pll->active = 0;
15315                 pll->config.crtc_mask = 0;
15316                 for_each_intel_crtc(dev, crtc) {
15317                         if (crtc->active && intel_crtc_to_shared_dpll(crtc) == pll) {
15318                                 pll->active++;
15319                                 pll->config.crtc_mask |= 1 << crtc->pipe;
15320                         }
15321                 }
15322
15323                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
15324                               pll->name, pll->config.crtc_mask, pll->on);
15325
15326                 if (pll->config.crtc_mask)
15327                         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
15328         }
15329
15330         for_each_intel_encoder(dev, encoder) {
15331                 pipe = 0;
15332
15333                 if (encoder->get_hw_state(encoder, &pipe)) {
15334                         crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
15335                         encoder->base.crtc = &crtc->base;
15336                         encoder->get_config(encoder, crtc->config);
15337                 } else {
15338                         encoder->base.crtc = NULL;
15339                 }
15340
15341                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
15342                               encoder->base.base.id,
15343                               encoder->base.name,
15344                               encoder->base.crtc ? "enabled" : "disabled",
15345                               pipe_name(pipe));
15346         }
15347
15348         for_each_intel_connector(dev, connector) {
15349                 if (connector->get_hw_state(connector)) {
15350                         connector->base.dpms = DRM_MODE_DPMS_ON;
15351                         connector->base.encoder = &connector->encoder->base;
15352                 } else {
15353                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15354                         connector->base.encoder = NULL;
15355                 }
15356                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
15357                               connector->base.base.id,
15358                               connector->base.name,
15359                               connector->base.encoder ? "enabled" : "disabled");
15360         }
15361
15362         for_each_intel_crtc(dev, crtc) {
15363                 crtc->base.hwmode = crtc->config->base.adjusted_mode;
15364
15365                 memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
15366                 if (crtc->base.state->active) {
15367                         intel_mode_from_pipe_config(&crtc->base.mode, crtc->config);
15368                         intel_mode_from_pipe_config(&crtc->base.state->adjusted_mode, crtc->config);
15369                         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
15370
15371                         /*
15372                          * The initial mode needs to be set in order to keep
15373                          * the atomic core happy. It wants a valid mode if the
15374                          * crtc's enabled, so we do the above call.
15375                          *
15376                          * At this point some state updated by the connectors
15377                          * in their ->detect() callback has not run yet, so
15378                          * no recalculation can be done yet.
15379                          *
15380                          * Even if we could do a recalculation and modeset
15381                          * right now it would cause a double modeset if
15382                          * fbdev or userspace chooses a different initial mode.
15383                          *
15384                          * If that happens, someone indicated they wanted a
15385                          * mode change, which means it's safe to do a full
15386                          * recalculation.
15387                          */
15388                         crtc->base.state->mode.private_flags = I915_MODE_FLAG_INHERITED;
15389
15390                         drm_calc_timestamping_constants(&crtc->base, &crtc->base.hwmode);
15391                         update_scanline_offset(crtc);
15392                 }
15393         }
15394 }
15395
15396 /* Scan out the current hw modeset state,
15397  * and sanitizes it to the current state
15398  */
15399 static void
15400 intel_modeset_setup_hw_state(struct drm_device *dev)
15401 {
15402         struct drm_i915_private *dev_priv = dev->dev_private;
15403         enum pipe pipe;
15404         struct intel_crtc *crtc;
15405         struct intel_encoder *encoder;
15406         int i;
15407
15408         intel_modeset_readout_hw_state(dev);
15409
15410         /* HW state is read out, now we need to sanitize this mess. */
15411         for_each_intel_encoder(dev, encoder) {
15412                 intel_sanitize_encoder(encoder);
15413         }
15414
15415         for_each_pipe(dev_priv, pipe) {
15416                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
15417                 intel_sanitize_crtc(crtc);
15418                 intel_dump_pipe_config(crtc, crtc->config,
15419                                        "[setup_hw_state]");
15420         }
15421
15422         intel_modeset_update_connector_atomic_state(dev);
15423
15424         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15425                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15426
15427                 if (!pll->on || pll->active)
15428                         continue;
15429
15430                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n", pll->name);
15431
15432                 pll->disable(dev_priv, pll);
15433                 pll->on = false;
15434         }
15435
15436         if (IS_VALLEYVIEW(dev))
15437                 vlv_wm_get_hw_state(dev);
15438         else if (IS_GEN9(dev))
15439                 skl_wm_get_hw_state(dev);
15440         else if (HAS_PCH_SPLIT(dev))
15441                 ilk_wm_get_hw_state(dev);
15442
15443         for_each_intel_crtc(dev, crtc) {
15444                 unsigned long put_domains;
15445
15446                 put_domains = modeset_get_crtc_power_domains(&crtc->base);
15447                 if (WARN_ON(put_domains))
15448                         modeset_put_power_domains(dev_priv, put_domains);
15449         }
15450         intel_display_set_init_power(dev_priv, false);
15451 }
15452
15453 void intel_display_resume(struct drm_device *dev)
15454 {
15455         struct drm_atomic_state *state = drm_atomic_state_alloc(dev);
15456         struct intel_connector *conn;
15457         struct intel_plane *plane;
15458         struct drm_crtc *crtc;
15459         int ret;
15460
15461         if (!state)
15462                 return;
15463
15464         state->acquire_ctx = dev->mode_config.acquire_ctx;
15465
15466         /* preserve complete old state, including dpll */
15467         intel_atomic_get_shared_dpll_state(state);
15468
15469         for_each_crtc(dev, crtc) {
15470                 struct drm_crtc_state *crtc_state =
15471                         drm_atomic_get_crtc_state(state, crtc);
15472
15473                 ret = PTR_ERR_OR_ZERO(crtc_state);
15474                 if (ret)
15475                         goto err;
15476
15477                 /* force a restore */
15478                 crtc_state->mode_changed = true;
15479         }
15480
15481         for_each_intel_plane(dev, plane) {
15482                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_plane_state(state, &plane->base));
15483                 if (ret)
15484                         goto err;
15485         }
15486
15487         for_each_intel_connector(dev, conn) {
15488                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(state, &conn->base));
15489                 if (ret)
15490                         goto err;
15491         }
15492
15493         intel_modeset_setup_hw_state(dev);
15494
15495         i915_redisable_vga(dev);
15496         ret = drm_atomic_commit(state);
15497         if (!ret)
15498                 return;
15499
15500 err:
15501         DRM_ERROR("Restoring old state failed with %i\n", ret);
15502         drm_atomic_state_free(state);
15503 }
15504
15505 void intel_modeset_gem_init(struct drm_device *dev)
15506 {
15507         struct drm_crtc *c;
15508         struct drm_i915_gem_object *obj;
15509         int ret;
15510
15511         mutex_lock(&dev->struct_mutex);
15512         intel_init_gt_powersave(dev);
15513         mutex_unlock(&dev->struct_mutex);
15514
15515         intel_modeset_init_hw(dev);
15516
15517         intel_setup_overlay(dev);
15518
15519         /*
15520          * Make sure any fbs we allocated at startup are properly
15521          * pinned & fenced.  When we do the allocation it's too early
15522          * for this.
15523          */
15524         for_each_crtc(dev, c) {
15525                 obj = intel_fb_obj(c->primary->fb);
15526                 if (obj == NULL)
15527                         continue;
15528
15529                 mutex_lock(&dev->struct_mutex);
15530                 ret = intel_pin_and_fence_fb_obj(c->primary,
15531                                                  c->primary->fb,
15532                                                  c->primary->state,
15533                                                  NULL, NULL);
15534                 mutex_unlock(&dev->struct_mutex);
15535                 if (ret) {
15536                         DRM_ERROR("failed to pin boot fb on pipe %d\n",
15537                                   to_intel_crtc(c)->pipe);
15538                         drm_framebuffer_unreference(c->primary->fb);
15539                         c->primary->fb = NULL;
15540                         c->primary->crtc = c->primary->state->crtc = NULL;
15541                         update_state_fb(c->primary);
15542                         c->state->plane_mask &= ~(1 << drm_plane_index(c->primary));
15543                 }
15544         }
15545
15546         intel_backlight_register(dev);
15547 }
15548
15549 void intel_connector_unregister(struct intel_connector *intel_connector)
15550 {
15551         struct drm_connector *connector = &intel_connector->base;
15552
15553         intel_panel_destroy_backlight(connector);
15554         drm_connector_unregister(connector);
15555 }
15556
15557 void intel_modeset_cleanup(struct drm_device *dev)
15558 {
15559         struct drm_i915_private *dev_priv = dev->dev_private;
15560         struct drm_connector *connector;
15561
15562         intel_disable_gt_powersave(dev);
15563
15564         intel_backlight_unregister(dev);
15565
15566         /*
15567          * Interrupts and polling as the first thing to avoid creating havoc.
15568          * Too much stuff here (turning of connectors, ...) would
15569          * experience fancy races otherwise.
15570          */
15571         intel_irq_uninstall(dev_priv);
15572
15573         /*
15574          * Due to the hpd irq storm handling the hotplug work can re-arm the
15575          * poll handlers. Hence disable polling after hpd handling is shut down.
15576          */
15577         drm_kms_helper_poll_fini(dev);
15578
15579         intel_unregister_dsm_handler();
15580
15581         intel_fbc_disable(dev_priv);
15582
15583         /* flush any delayed tasks or pending work */
15584         flush_scheduled_work();
15585
15586         /* destroy the backlight and sysfs files before encoders/connectors */
15587         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
15588                 struct intel_connector *intel_connector;
15589
15590                 intel_connector = to_intel_connector(connector);
15591                 intel_connector->unregister(intel_connector);
15592         }
15593
15594         drm_mode_config_cleanup(dev);
15595
15596         intel_cleanup_overlay(dev);
15597
15598         mutex_lock(&dev->struct_mutex);
15599         intel_cleanup_gt_powersave(dev);
15600         mutex_unlock(&dev->struct_mutex);
15601
15602         intel_teardown_gmbus(dev);
15603 }
15604
15605 /*
15606  * Return which encoder is currently attached for connector.
15607  */
15608 struct drm_encoder *intel_best_encoder(struct drm_connector *connector)
15609 {
15610         return &intel_attached_encoder(connector)->base;
15611 }
15612
15613 void intel_connector_attach_encoder(struct intel_connector *connector,
15614                                     struct intel_encoder *encoder)
15615 {
15616         connector->encoder = encoder;
15617         drm_mode_connector_attach_encoder(&connector->base,
15618                                           &encoder->base);
15619 }
15620
15621 /*
15622  * set vga decode state - true == enable VGA decode
15623  */
15624 int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
15625 {
15626         struct drm_i915_private *dev_priv = dev->dev_private;
15627         unsigned reg = INTEL_INFO(dev)->gen >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
15628         u16 gmch_ctrl;
15629
15630         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
15631                 DRM_ERROR("failed to read control word\n");
15632                 return -EIO;
15633         }
15634
15635         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
15636                 return 0;
15637
15638         if (state)
15639                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
15640         else
15641                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
15642
15643         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
15644                 DRM_ERROR("failed to write control word\n");
15645                 return -EIO;
15646         }
15647
15648         return 0;
15649 }
15650
15651 struct intel_display_error_state {
15652
15653         u32 power_well_driver;
15654
15655         int num_transcoders;
15656
15657         struct intel_cursor_error_state {
15658                 u32 control;
15659                 u32 position;
15660                 u32 base;
15661                 u32 size;
15662         } cursor[I915_MAX_PIPES];
15663
15664         struct intel_pipe_error_state {
15665                 bool power_domain_on;
15666                 u32 source;
15667                 u32 stat;
15668         } pipe[I915_MAX_PIPES];
15669
15670         struct intel_plane_error_state {
15671                 u32 control;
15672                 u32 stride;
15673                 u32 size;
15674                 u32 pos;
15675                 u32 addr;
15676                 u32 surface;
15677                 u32 tile_offset;
15678         } plane[I915_MAX_PIPES];
15679
15680         struct intel_transcoder_error_state {
15681                 bool power_domain_on;
15682                 enum transcoder cpu_transcoder;
15683
15684                 u32 conf;
15685
15686                 u32 htotal;
15687                 u32 hblank;
15688                 u32 hsync;
15689                 u32 vtotal;
15690                 u32 vblank;
15691                 u32 vsync;
15692         } transcoder[4];
15693 };
15694
15695 struct intel_display_error_state *
15696 intel_display_capture_error_state(struct drm_device *dev)
15697 {
15698         struct drm_i915_private *dev_priv = dev->dev_private;
15699         struct intel_display_error_state *error;
15700         int transcoders[] = {
15701                 TRANSCODER_A,
15702                 TRANSCODER_B,
15703                 TRANSCODER_C,
15704                 TRANSCODER_EDP,
15705         };
15706         int i;
15707
15708         if (INTEL_INFO(dev)->num_pipes == 0)
15709                 return NULL;
15710
15711         error = kzalloc(sizeof(*error), GFP_ATOMIC);
15712         if (error == NULL)
15713                 return NULL;
15714
15715         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
15716                 error->power_well_driver = I915_READ(HSW_PWR_WELL_DRIVER);
15717
15718         for_each_pipe(dev_priv, i) {
15719                 error->pipe[i].power_domain_on =
15720                         __intel_display_power_is_enabled(dev_priv,
15721                                                          POWER_DOMAIN_PIPE(i));
15722                 if (!error->pipe[i].power_domain_on)
15723                         continue;
15724
15725                 error->cursor[i].control = I915_READ(CURCNTR(i));
15726                 error->cursor[i].position = I915_READ(CURPOS(i));
15727                 error->cursor[i].base = I915_READ(CURBASE(i));
15728
15729                 error->plane[i].control = I915_READ(DSPCNTR(i));
15730                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
15731                 if (INTEL_INFO(dev)->gen <= 3) {
15732                         error->plane[i].size = I915_READ(DSPSIZE(i));
15733                         error->plane[i].pos = I915_READ(DSPPOS(i));
15734                 }
15735                 if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
15736                         error->plane[i].addr = I915_READ(DSPADDR(i));
15737                 if (INTEL_INFO(dev)->gen >= 4) {
15738                         error->plane[i].surface = I915_READ(DSPSURF(i));
15739                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
15740                 }
15741
15742                 error->pipe[i].source = I915_READ(PIPESRC(i));
15743
15744                 if (HAS_GMCH_DISPLAY(dev))
15745                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
15746         }
15747
15748         error->num_transcoders = INTEL_INFO(dev)->num_pipes;
15749         if (HAS_DDI(dev_priv->dev))
15750                 error->num_transcoders++; /* Account for eDP. */
15751
15752         for (i = 0; i < error->num_transcoders; i++) {
15753                 enum transcoder cpu_transcoder = transcoders[i];
15754
15755                 error->transcoder[i].power_domain_on =
15756                         __intel_display_power_is_enabled(dev_priv,
15757                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
15758                 if (!error->transcoder[i].power_domain_on)
15759                         continue;
15760
15761                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
15762
15763                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
15764                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
15765                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
15766                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
15767                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
15768                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
15769                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
15770         }
15771
15772         return error;
15773 }
15774
15775 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
15776
15777 void
15778 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
15779                                 struct drm_device *dev,
15780                                 struct intel_display_error_state *error)
15781 {
15782         struct drm_i915_private *dev_priv = dev->dev_private;
15783         int i;
15784
15785         if (!error)
15786                 return;
15787
15788         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev)->num_pipes);
15789         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
15790                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
15791                            error->power_well_driver);
15792         for_each_pipe(dev_priv, i) {
15793                 err_printf(m, "Pipe [%d]:\n", i);
15794                 err_printf(m, "  Power: %s\n",
15795                            error->pipe[i].power_domain_on ? "on" : "off");
15796                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
15797                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
15798
15799                 err_printf(m, "Plane [%d]:\n", i);
15800                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
15801                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
15802                 if (INTEL_INFO(dev)->gen <= 3) {
15803                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
15804                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
15805                 }
15806                 if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
15807                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
15808                 if (INTEL_INFO(dev)->gen >= 4) {
15809                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
15810                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
15811                 }
15812
15813                 err_printf(m, "Cursor [%d]:\n", i);
15814                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
15815                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
15816                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
15817         }
15818
15819         for (i = 0; i < error->num_transcoders; i++) {
15820                 err_printf(m, "CPU transcoder: %c\n",
15821                            transcoder_name(error->transcoder[i].cpu_transcoder));
15822                 err_printf(m, "  Power: %s\n",
15823                            error->transcoder[i].power_domain_on ? "on" : "off");
15824                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
15825                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
15826                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
15827                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
15828                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
15829                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
15830                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
15831         }
15832 }
15833
15834 void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file)
15835 {
15836         struct intel_crtc *crtc;
15837
15838         for_each_intel_crtc(dev, crtc) {
15839                 struct intel_unpin_work *work;
15840
15841                 spin_lock_irq(&dev->event_lock);
15842
15843                 work = crtc->unpin_work;
15844
15845                 if (work && work->event &&
15846                     work->event->base.file_priv == file) {
15847                         kfree(work->event);
15848                         work->event = NULL;
15849                 }
15850
15851                 spin_unlock_irq(&dev->event_lock);
15852         }
15853 }