drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/cpufreq.h>
  29 #include "i915_drv.h"
  30 #include "intel_drv.h"
  31 #include "../../../platform/x86/intel_ips.h"
  32 #include <linux/module.h>
  33
  34 /**
  35  * RC6 is a special power stage which allows the GPU to enter an very
  36  * low-voltage mode when idle, using down to 0V while at this stage.  This
  37  * stage is entered automatically when the GPU is idle when RC6 support is
  38  * enabled, and as soon as new workload arises GPU wakes up automatically as well.
  39  *
  40  * There are different RC6 modes available in Intel GPU, which differentiate
  41  * among each other with the latency required to enter and leave RC6 and
  42  * voltage consumed by the GPU in different states.
  43  *
  44  * The combination of the following flags define which states GPU is allowed
  45  * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
  46  * RC6pp is deepest RC6. Their support by hardware varies according to the
  47  * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
  48  * which brings the most power savings; deeper states save more power, but
  49  * require higher latency to switch to and wake up.
  50  */
  51 #define INTEL_RC6_ENABLE                        (1<<0)
  52 #define INTEL_RC6p_ENABLE                       (1<<1)
  53 #define INTEL_RC6pp_ENABLE                      (1<<2)
  54
  55 static void gen9_init_clock_gating(struct drm_device *dev)
  56 {
  57         struct drm_i915_private *dev_priv = dev->dev_private;
  58
  59         /* WaEnableLbsSlaRetryTimerDecrement:skl */
  60         I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
  61                    GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
  62
  63         /* WaDisableKillLogic:bxt,skl */
  64         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
  65                    ECOCHK_DIS_TLB);
  66 }
  67
  68 static void skl_init_clock_gating(struct drm_device *dev)
  69 {
  70         struct drm_i915_private *dev_priv = dev->dev_private;
  71
  72         gen9_init_clock_gating(dev);
  73
  74         if (INTEL_REVID(dev) <= SKL_REVID_B0) {
  75                 /*
  76                  * WaDisableSDEUnitClockGating:skl
  77                  * WaSetGAPSunitClckGateDisable:skl
  78                  */
  79                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
  80                            GEN8_GAPSUNIT_CLOCK_GATE_DISABLE |
  81                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  82
  83                 /* WaDisableVFUnitClockGating:skl */
  84                 I915_WRITE(GEN6_UCGCTL2, I915_READ(GEN6_UCGCTL2) |
  85                            GEN6_VFUNIT_CLOCK_GATE_DISABLE);
  86         }
  87
  88         if (INTEL_REVID(dev) <= SKL_REVID_D0) {
  89                 /* WaDisableHDCInvalidation:skl */
  90                 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
  91                            BDW_DISABLE_HDC_INVALIDATION);
  92
  93                 /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
  94                 I915_WRITE(FF_SLICE_CS_CHICKEN2,
  95                            _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
  96         }
  97
  98         /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
  99          * involving this register should also be added to WA batch as required.
 100          */
 101         if (INTEL_REVID(dev) <= SKL_REVID_E0)
 102                 /* WaDisableLSQCROPERFforOCL:skl */
 103                 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
 104                            GEN8_LQSC_RO_PERF_DIS);
 105
 106         /* WaEnableGapsTsvCreditFix:skl */
 107         if (IS_SKYLAKE(dev) && (INTEL_REVID(dev) >= SKL_REVID_C0)) {
 108                 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
 109                                            GEN9_GAPS_TSV_CREDIT_DISABLE));
 110         }
 111 }
 112
 113 static void bxt_init_clock_gating(struct drm_device *dev)
 114 {
 115         struct drm_i915_private *dev_priv = dev->dev_private;
 116
 117         gen9_init_clock_gating(dev);
 118
 119         /* WaDisableSDEUnitClockGating:bxt */
 120         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 121                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 122
 123         /*
 124          * FIXME:
 125          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 126          */
 127         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 128                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 129
 130         if (INTEL_REVID(dev) == BXT_REVID_A0) {
 131                 /*
 132                  * Hardware specification requires this bit to be
 133                  * set to 1 for A0
 134                  */
 135                 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
 136         }
 137
 138         /* WaSetClckGatingDisableMedia:bxt */
 139         if (INTEL_REVID(dev) == BXT_REVID_A0) {
 140                 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
 141                                             ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
 142         }
 143 }
 144
 145 static void i915_pineview_get_mem_freq(struct drm_device *dev)
 146 {
 147         struct drm_i915_private *dev_priv = dev->dev_private;
 148         u32 tmp;
 149
 150         tmp = I915_READ(CLKCFG);
 151
 152         switch (tmp & CLKCFG_FSB_MASK) {
 153         case CLKCFG_FSB_533:
 154                 dev_priv->fsb_freq = 533; /* 133*4 */
 155                 break;
 156         case CLKCFG_FSB_800:
 157                 dev_priv->fsb_freq = 800; /* 200*4 */
 158                 break;
 159         case CLKCFG_FSB_667:
 160                 dev_priv->fsb_freq =  667; /* 167*4 */
 161                 break;
 162         case CLKCFG_FSB_400:
 163                 dev_priv->fsb_freq = 400; /* 100*4 */
 164                 break;
 165         }
 166
 167         switch (tmp & CLKCFG_MEM_MASK) {
 168         case CLKCFG_MEM_533:
 169                 dev_priv->mem_freq = 533;
 170                 break;
 171         case CLKCFG_MEM_667:
 172                 dev_priv->mem_freq = 667;
 173                 break;
 174         case CLKCFG_MEM_800:
 175                 dev_priv->mem_freq = 800;
 176                 break;
 177         }
 178
 179         /* detect pineview DDR3 setting */
 180         tmp = I915_READ(CSHRDDR3CTL);
 181         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 182 }
 183
 184 static void i915_ironlake_get_mem_freq(struct drm_device *dev)
 185 {
 186         struct drm_i915_private *dev_priv = dev->dev_private;
 187         u16 ddrpll, csipll;
 188
 189         ddrpll = I915_READ16(DDRMPLL1);
 190         csipll = I915_READ16(CSIPLL0);
 191
 192         switch (ddrpll & 0xff) {
 193         case 0xc:
 194                 dev_priv->mem_freq = 800;
 195                 break;
 196         case 0x10:
 197                 dev_priv->mem_freq = 1066;
 198                 break;
 199         case 0x14:
 200                 dev_priv->mem_freq = 1333;
 201                 break;
 202         case 0x18:
 203                 dev_priv->mem_freq = 1600;
 204                 break;
 205         default:
 206                 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
 207                                  ddrpll & 0xff);
 208                 dev_priv->mem_freq = 0;
 209                 break;
 210         }
 211
 212         dev_priv->ips.r_t = dev_priv->mem_freq;
 213
 214         switch (csipll & 0x3ff) {
 215         case 0x00c:
 216                 dev_priv->fsb_freq = 3200;
 217                 break;
 218         case 0x00e:
 219                 dev_priv->fsb_freq = 3733;
 220                 break;
 221         case 0x010:
 222                 dev_priv->fsb_freq = 4266;
 223                 break;
 224         case 0x012:
 225                 dev_priv->fsb_freq = 4800;
 226                 break;
 227         case 0x014:
 228                 dev_priv->fsb_freq = 5333;
 229                 break;
 230         case 0x016:
 231                 dev_priv->fsb_freq = 5866;
 232                 break;
 233         case 0x018:
 234                 dev_priv->fsb_freq = 6400;
 235                 break;
 236         default:
 237                 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
 238                                  csipll & 0x3ff);
 239                 dev_priv->fsb_freq = 0;
 240                 break;
 241         }
 242
 243         if (dev_priv->fsb_freq == 3200) {
 244                 dev_priv->ips.c_m = 0;
 245         } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
 246                 dev_priv->ips.c_m = 1;
 247         } else {
 248                 dev_priv->ips.c_m = 2;
 249         }
 250 }
 251
 252 static const struct cxsr_latency cxsr_latency_table[] = {
 253         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 254         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 255         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 256         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 257         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 258
 259         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 260         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 261         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 262         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 263         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 264
 265         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 266         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 267         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 268         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 269         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 270
 271         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 272         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 273         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 274         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 275         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 276
 277         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 278         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 279         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 280         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 281         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 282
 283         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 284         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 285         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 286         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 287         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 288 };
 289
 290 static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
 291                                                          int is_ddr3,
 292                                                          int fsb,
 293                                                          int mem)
 294 {
 295         const struct cxsr_latency *latency;
 296         int i;
 297
 298         if (fsb == 0 || mem == 0)
 299                 return NULL;
 300
 301         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 302                 latency = &cxsr_latency_table[i];
 303                 if (is_desktop == latency->is_desktop &&
 304                     is_ddr3 == latency->is_ddr3 &&
 305                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 306                         return latency;
 307         }
 308
 309         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 310
 311         return NULL;
 312 }
 313
 314 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 315 {
 316         u32 val;
 317
 318         mutex_lock(&dev_priv->rps.hw_lock);
 319
 320         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 321         if (enable)
 322                 val &= ~FORCE_DDR_HIGH_FREQ;
 323         else
 324                 val |= FORCE_DDR_HIGH_FREQ;
 325         val &= ~FORCE_DDR_LOW_FREQ;
 326         val |= FORCE_DDR_FREQ_REQ_ACK;
 327         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 328
 329         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 330                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 331                 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
 332
 333         mutex_unlock(&dev_priv->rps.hw_lock);
 334 }
 335
 336 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 337 {
 338         u32 val;
 339
 340         mutex_lock(&dev_priv->rps.hw_lock);
 341
 342         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
 343         if (enable)
 344                 val |= DSP_MAXFIFO_PM5_ENABLE;
 345         else
 346                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 347         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
 348
 349         mutex_unlock(&dev_priv->rps.hw_lock);
 350 }
 351
 352 #define FW_WM(value, plane) \
 353         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 354
 355 void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 356 {
 357         struct drm_device *dev = dev_priv->dev;
 358         u32 val;
 359
 360         if (IS_VALLEYVIEW(dev)) {
 361                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 362                 POSTING_READ(FW_BLC_SELF_VLV);
 363                 dev_priv->wm.vlv.cxsr = enable;
 364         } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) {
 365                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 366                 POSTING_READ(FW_BLC_SELF);
 367         } else if (IS_PINEVIEW(dev)) {
 368                 val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
 369                 val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
 370                 I915_WRITE(DSPFW3, val);
 371                 POSTING_READ(DSPFW3);
 372         } else if (IS_I945G(dev) || IS_I945GM(dev)) {
 373                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 374                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 375                 I915_WRITE(FW_BLC_SELF, val);
 376                 POSTING_READ(FW_BLC_SELF);
 377         } else if (IS_I915GM(dev)) {
 378                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 379                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 380                 I915_WRITE(INSTPM, val);
 381                 POSTING_READ(INSTPM);
 382         } else {
 383                 return;
 384         }
 385
 386         DRM_DEBUG_KMS("memory self-refresh is %s\n",
 387                       enable ? "enabled" : "disabled");
 388 }
 389
 390
 391 /*
 392  * Latency for FIFO fetches is dependent on several factors:
 393  *   - memory configuration (speed, channels)
 394  *   - chipset
 395  *   - current MCH state
 396  * It can be fairly high in some situations, so here we assume a fairly
 397  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 398  * set this value too high, the FIFO will fetch frequently to stay full)
 399  * and power consumption (set it too low to save power and we might see
 400  * FIFO underruns and display "flicker").
 401  *
 402  * A value of 5us seems to be a good balance; safe for very low end
 403  * platforms but not overly aggressive on lower latency configs.
 404  */
 405 static const int pessimal_latency_ns = 5000;
 406
 407 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 408         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 409
 410 static int vlv_get_fifo_size(struct drm_device *dev,
 411                               enum pipe pipe, int plane)
 412 {
 413         struct drm_i915_private *dev_priv = dev->dev_private;
 414         int sprite0_start, sprite1_start, size;
 415
 416         switch (pipe) {
 417                 uint32_t dsparb, dsparb2, dsparb3;
 418         case PIPE_A:
 419                 dsparb = I915_READ(DSPARB);
 420                 dsparb2 = I915_READ(DSPARB2);
 421                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 422                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 423                 break;
 424         case PIPE_B:
 425                 dsparb = I915_READ(DSPARB);
 426                 dsparb2 = I915_READ(DSPARB2);
 427                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 428                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 429                 break;
 430         case PIPE_C:
 431                 dsparb2 = I915_READ(DSPARB2);
 432                 dsparb3 = I915_READ(DSPARB3);
 433                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 434                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 435                 break;
 436         default:
 437                 return 0;
 438         }
 439
 440         switch (plane) {
 441         case 0:
 442                 size = sprite0_start;
 443                 break;
 444         case 1:
 445                 size = sprite1_start - sprite0_start;
 446                 break;
 447         case 2:
 448                 size = 512 - 1 - sprite1_start;
 449                 break;
 450         default:
 451                 return 0;
 452         }
 453
 454         DRM_DEBUG_KMS("Pipe %c %s %c FIFO size: %d\n",
 455                       pipe_name(pipe), plane == 0 ? "primary" : "sprite",
 456                       plane == 0 ? plane_name(pipe) : sprite_name(pipe, plane - 1),
 457                       size);
 458
 459         return size;
 460 }
 461
 462 static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
 463 {
 464         struct drm_i915_private *dev_priv = dev->dev_private;
 465         uint32_t dsparb = I915_READ(DSPARB);
 466         int size;
 467
 468         size = dsparb & 0x7f;
 469         if (plane)
 470                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 471
 472         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 473                       plane ? "B" : "A", size);
 474
 475         return size;
 476 }
 477
 478 static int i830_get_fifo_size(struct drm_device *dev, int plane)
 479 {
 480         struct drm_i915_private *dev_priv = dev->dev_private;
 481         uint32_t dsparb = I915_READ(DSPARB);
 482         int size;
 483
 484         size = dsparb & 0x1ff;
 485         if (plane)
 486                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 487         size >>= 1; /* Convert to cachelines */
 488
 489         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 490                       plane ? "B" : "A", size);
 491
 492         return size;
 493 }
 494
 495 static int i845_get_fifo_size(struct drm_device *dev, int plane)
 496 {
 497         struct drm_i915_private *dev_priv = dev->dev_private;
 498         uint32_t dsparb = I915_READ(DSPARB);
 499         int size;
 500
 501         size = dsparb & 0x7f;
 502         size >>= 2; /* Convert to cachelines */
 503
 504         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 505                       plane ? "B" : "A",
 506                       size);
 507
 508         return size;
 509 }
 510
 511 /* Pineview has different values for various configs */
 512 static const struct intel_watermark_params pineview_display_wm = {
 513         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 514         .max_wm = PINEVIEW_MAX_WM,
 515         .default_wm = PINEVIEW_DFT_WM,
 516         .guard_size = PINEVIEW_GUARD_WM,
 517         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 518 };
 519 static const struct intel_watermark_params pineview_display_hplloff_wm = {
 520         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 521         .max_wm = PINEVIEW_MAX_WM,
 522         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 523         .guard_size = PINEVIEW_GUARD_WM,
 524         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 525 };
 526 static const struct intel_watermark_params pineview_cursor_wm = {
 527         .fifo_size = PINEVIEW_CURSOR_FIFO,
 528         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 529         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 530         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 531         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 532 };
 533 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
 534         .fifo_size = PINEVIEW_CURSOR_FIFO,
 535         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 536         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 537         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 538         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 539 };
 540 static const struct intel_watermark_params g4x_wm_info = {
 541         .fifo_size = G4X_FIFO_SIZE,
 542         .max_wm = G4X_MAX_WM,
 543         .default_wm = G4X_MAX_WM,
 544         .guard_size = 2,
 545         .cacheline_size = G4X_FIFO_LINE_SIZE,
 546 };
 547 static const struct intel_watermark_params g4x_cursor_wm_info = {
 548         .fifo_size = I965_CURSOR_FIFO,
 549         .max_wm = I965_CURSOR_MAX_WM,
 550         .default_wm = I965_CURSOR_DFT_WM,
 551         .guard_size = 2,
 552         .cacheline_size = G4X_FIFO_LINE_SIZE,
 553 };
 554 static const struct intel_watermark_params valleyview_wm_info = {
 555         .fifo_size = VALLEYVIEW_FIFO_SIZE,
 556         .max_wm = VALLEYVIEW_MAX_WM,
 557         .default_wm = VALLEYVIEW_MAX_WM,
 558         .guard_size = 2,
 559         .cacheline_size = G4X_FIFO_LINE_SIZE,
 560 };
 561 static const struct intel_watermark_params valleyview_cursor_wm_info = {
 562         .fifo_size = I965_CURSOR_FIFO,
 563         .max_wm = VALLEYVIEW_CURSOR_MAX_WM,
 564         .default_wm = I965_CURSOR_DFT_WM,
 565         .guard_size = 2,
 566         .cacheline_size = G4X_FIFO_LINE_SIZE,
 567 };
 568 static const struct intel_watermark_params i965_cursor_wm_info = {
 569         .fifo_size = I965_CURSOR_FIFO,
 570         .max_wm = I965_CURSOR_MAX_WM,
 571         .default_wm = I965_CURSOR_DFT_WM,
 572         .guard_size = 2,
 573         .cacheline_size = I915_FIFO_LINE_SIZE,
 574 };
 575 static const struct intel_watermark_params i945_wm_info = {
 576         .fifo_size = I945_FIFO_SIZE,
 577         .max_wm = I915_MAX_WM,
 578         .default_wm = 1,
 579         .guard_size = 2,
 580         .cacheline_size = I915_FIFO_LINE_SIZE,
 581 };
 582 static const struct intel_watermark_params i915_wm_info = {
 583         .fifo_size = I915_FIFO_SIZE,
 584         .max_wm = I915_MAX_WM,
 585         .default_wm = 1,
 586         .guard_size = 2,
 587         .cacheline_size = I915_FIFO_LINE_SIZE,
 588 };
 589 static const struct intel_watermark_params i830_a_wm_info = {
 590         .fifo_size = I855GM_FIFO_SIZE,
 591         .max_wm = I915_MAX_WM,
 592         .default_wm = 1,
 593         .guard_size = 2,
 594         .cacheline_size = I830_FIFO_LINE_SIZE,
 595 };
 596 static const struct intel_watermark_params i830_bc_wm_info = {
 597         .fifo_size = I855GM_FIFO_SIZE,
 598         .max_wm = I915_MAX_WM/2,
 599         .default_wm = 1,
 600         .guard_size = 2,
 601         .cacheline_size = I830_FIFO_LINE_SIZE,
 602 };
 603 static const struct intel_watermark_params i845_wm_info = {
 604         .fifo_size = I830_FIFO_SIZE,
 605         .max_wm = I915_MAX_WM,
 606         .default_wm = 1,
 607         .guard_size = 2,
 608         .cacheline_size = I830_FIFO_LINE_SIZE,
 609 };
 610
 611 /**
 612  * intel_calculate_wm - calculate watermark level
 613  * @clock_in_khz: pixel clock
 614  * @wm: chip FIFO params
 615  * @pixel_size: display pixel size
 616  * @latency_ns: memory latency for the platform
 617  *
 618  * Calculate the watermark level (the level at which the display plane will
 619  * start fetching from memory again).  Each chip has a different display
 620  * FIFO size and allocation, so the caller needs to figure that out and pass
 621  * in the correct intel_watermark_params structure.
 622  *
 623  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 624  * on the pixel size.  When it reaches the watermark level, it'll start
 625  * fetching FIFO line sized based chunks from memory until the FIFO fills
 626  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 627  * will occur, and a display engine hang could result.
 628  */
 629 static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
 630                                         const struct intel_watermark_params *wm,
 631                                         int fifo_size,
 632                                         int pixel_size,
 633                                         unsigned long latency_ns)
 634 {
 635         long entries_required, wm_size;
 636
 637         /*
 638          * Note: we need to make sure we don't overflow for various clock &
 639          * latency values.
 640          * clocks go from a few thousand to several hundred thousand.
 641          * latency is usually a few thousand
 642          */
 643         entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
 644                 1000;
 645         entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
 646
 647         DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
 648
 649         wm_size = fifo_size - (entries_required + wm->guard_size);
 650
 651         DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
 652
 653         /* Don't promote wm_size to unsigned... */
 654         if (wm_size > (long)wm->max_wm)
 655                 wm_size = wm->max_wm;
 656         if (wm_size <= 0)
 657                 wm_size = wm->default_wm;
 658
 659         /*
 660          * Bspec seems to indicate that the value shouldn't be lower than
 661          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 662          * Lets go for 8 which is the burst size since certain platforms
 663          * already use a hardcoded 8 (which is what the spec says should be
 664          * done).
 665          */
 666         if (wm_size <= 8)
 667                 wm_size = 8;
 668
 669         return wm_size;
 670 }
 671
 672 static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
 673 {
 674         struct drm_crtc *crtc, *enabled = NULL;
 675
 676         for_each_crtc(dev, crtc) {
 677                 if (intel_crtc_active(crtc)) {
 678                         if (enabled)
 679                                 return NULL;
 680                         enabled = crtc;
 681                 }
 682         }
 683
 684         return enabled;
 685 }
 686
 687 static void pineview_update_wm(struct drm_crtc *unused_crtc)
 688 {
 689         struct drm_device *dev = unused_crtc->dev;
 690         struct drm_i915_private *dev_priv = dev->dev_private;
 691         struct drm_crtc *crtc;
 692         const struct cxsr_latency *latency;
 693         u32 reg;
 694         unsigned long wm;
 695
 696         latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
 697                                          dev_priv->fsb_freq, dev_priv->mem_freq);
 698         if (!latency) {
 699                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 700                 intel_set_memory_cxsr(dev_priv, false);
 701                 return;
 702         }
 703
 704         crtc = single_enabled_crtc(dev);
 705         if (crtc) {
 706                 const struct drm_display_mode *adjusted_mode;
 707                 int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
 708                 int clock;
 709
 710                 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
 711                 clock = adjusted_mode->crtc_clock;
 712
 713                 /* Display SR */
 714                 wm = intel_calculate_wm(clock, &pineview_display_wm,
 715                                         pineview_display_wm.fifo_size,
 716                                         pixel_size, latency->display_sr);
 717                 reg = I915_READ(DSPFW1);
 718                 reg &= ~DSPFW_SR_MASK;
 719                 reg |= FW_WM(wm, SR);
 720                 I915_WRITE(DSPFW1, reg);
 721                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
 722
 723                 /* cursor SR */
 724                 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
 725                                         pineview_display_wm.fifo_size,
 726                                         pixel_size, latency->cursor_sr);
 727                 reg = I915_READ(DSPFW3);
 728                 reg &= ~DSPFW_CURSOR_SR_MASK;
 729                 reg |= FW_WM(wm, CURSOR_SR);
 730                 I915_WRITE(DSPFW3, reg);
 731
 732                 /* Display HPLL off SR */
 733                 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
 734                                         pineview_display_hplloff_wm.fifo_size,
 735                                         pixel_size, latency->display_hpll_disable);
 736                 reg = I915_READ(DSPFW3);
 737                 reg &= ~DSPFW_HPLL_SR_MASK;
 738                 reg |= FW_WM(wm, HPLL_SR);
 739                 I915_WRITE(DSPFW3, reg);
 740
 741                 /* cursor HPLL off SR */
 742                 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
 743                                         pineview_display_hplloff_wm.fifo_size,
 744                                         pixel_size, latency->cursor_hpll_disable);
 745                 reg = I915_READ(DSPFW3);
 746                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 747                 reg |= FW_WM(wm, HPLL_CURSOR);
 748                 I915_WRITE(DSPFW3, reg);
 749                 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
 750
 751                 intel_set_memory_cxsr(dev_priv, true);
 752         } else {
 753                 intel_set_memory_cxsr(dev_priv, false);
 754         }
 755 }
 756
 757 static bool g4x_compute_wm0(struct drm_device *dev,
 758                             int plane,
 759                             const struct intel_watermark_params *display,
 760                             int display_latency_ns,
 761                             const struct intel_watermark_params *cursor,
 762                             int cursor_latency_ns,
 763                             int *plane_wm,
 764                             int *cursor_wm)
 765 {
 766         struct drm_crtc *crtc;
 767         const struct drm_display_mode *adjusted_mode;
 768         int htotal, hdisplay, clock, pixel_size;
 769         int line_time_us, line_count;
 770         int entries, tlb_miss;
 771
 772         crtc = intel_get_crtc_for_plane(dev, plane);
 773         if (!intel_crtc_active(crtc)) {
 774                 *cursor_wm = cursor->guard_size;
 775                 *plane_wm = display->guard_size;
 776                 return false;
 777         }
 778
 779         adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
 780         clock = adjusted_mode->crtc_clock;
 781         htotal = adjusted_mode->crtc_htotal;
 782         hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
 783         pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
 784
 785         /* Use the small buffer method to calculate plane watermark */
 786         entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
 787         tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
 788         if (tlb_miss > 0)
 789                 entries += tlb_miss;
 790         entries = DIV_ROUND_UP(entries, display->cacheline_size);
 791         *plane_wm = entries + display->guard_size;
 792         if (*plane_wm > (int)display->max_wm)
 793                 *plane_wm = display->max_wm;
 794
 795         /* Use the large buffer method to calculate cursor watermark */
 796         line_time_us = max(htotal * 1000 / clock, 1);
 797         line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
 798         entries = line_count * crtc->cursor->state->crtc_w * pixel_size;
 799         tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
 800         if (tlb_miss > 0)
 801                 entries += tlb_miss;
 802         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
 803         *cursor_wm = entries + cursor->guard_size;
 804         if (*cursor_wm > (int)cursor->max_wm)
 805                 *cursor_wm = (int)cursor->max_wm;
 806
 807         return true;
 808 }
 809
 810 /*
 811  * Check the wm result.
 812  *
 813  * If any calculated watermark values is larger than the maximum value that
 814  * can be programmed into the associated watermark register, that watermark
 815  * must be disabled.
 816  */
 817 static bool g4x_check_srwm(struct drm_device *dev,
 818                            int display_wm, int cursor_wm,
 819                            const struct intel_watermark_params *display,
 820                            const struct intel_watermark_params *cursor)
 821 {
 822         DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
 823                       display_wm, cursor_wm);
 824
 825         if (display_wm > display->max_wm) {
 826                 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
 827                               display_wm, display->max_wm);
 828                 return false;
 829         }
 830
 831         if (cursor_wm > cursor->max_wm) {
 832                 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
 833                               cursor_wm, cursor->max_wm);
 834                 return false;
 835         }
 836
 837         if (!(display_wm || cursor_wm)) {
 838                 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
 839                 return false;
 840         }
 841
 842         return true;
 843 }
 844
 845 static bool g4x_compute_srwm(struct drm_device *dev,
 846                              int plane,
 847                              int latency_ns,
 848                              const struct intel_watermark_params *display,
 849                              const struct intel_watermark_params *cursor,
 850                              int *display_wm, int *cursor_wm)
 851 {
 852         struct drm_crtc *crtc;
 853         const struct drm_display_mode *adjusted_mode;
 854         int hdisplay, htotal, pixel_size, clock;
 855         unsigned long line_time_us;
 856         int line_count, line_size;
 857         int small, large;
 858         int entries;
 859
 860         if (!latency_ns) {
 861                 *display_wm = *cursor_wm = 0;
 862                 return false;
 863         }
 864
 865         crtc = intel_get_crtc_for_plane(dev, plane);
 866         adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
 867         clock = adjusted_mode->crtc_clock;
 868         htotal = adjusted_mode->crtc_htotal;
 869         hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
 870         pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
 871
 872         line_time_us = max(htotal * 1000 / clock, 1);
 873         line_count = (latency_ns / line_time_us + 1000) / 1000;
 874         line_size = hdisplay * pixel_size;
 875
 876         /* Use the minimum of the small and large buffer method for primary */
 877         small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
 878         large = line_count * line_size;
 879
 880         entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
 881         *display_wm = entries + display->guard_size;
 882
 883         /* calculate the self-refresh watermark for display cursor */
 884         entries = line_count * pixel_size * crtc->cursor->state->crtc_w;
 885         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
 886         *cursor_wm = entries + cursor->guard_size;
 887
 888         return g4x_check_srwm(dev,
 889                               *display_wm, *cursor_wm,
 890                               display, cursor);
 891 }
 892
 893 #define FW_WM_VLV(value, plane) \
 894         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 895
 896 static void vlv_write_wm_values(struct intel_crtc *crtc,
 897                                 const struct vlv_wm_values *wm)
 898 {
 899         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 900         enum pipe pipe = crtc->pipe;
 901
 902         I915_WRITE(VLV_DDL(pipe),
 903                    (wm->ddl[pipe].cursor << DDL_CURSOR_SHIFT) |
 904                    (wm->ddl[pipe].sprite[1] << DDL_SPRITE_SHIFT(1)) |
 905                    (wm->ddl[pipe].sprite[0] << DDL_SPRITE_SHIFT(0)) |
 906                    (wm->ddl[pipe].primary << DDL_PLANE_SHIFT));
 907
 908         I915_WRITE(DSPFW1,
 909                    FW_WM(wm->sr.plane, SR) |
 910                    FW_WM(wm->pipe[PIPE_B].cursor, CURSORB) |
 911                    FW_WM_VLV(wm->pipe[PIPE_B].primary, PLANEB) |
 912                    FW_WM_VLV(wm->pipe[PIPE_A].primary, PLANEA));
 913         I915_WRITE(DSPFW2,
 914                    FW_WM_VLV(wm->pipe[PIPE_A].sprite[1], SPRITEB) |
 915                    FW_WM(wm->pipe[PIPE_A].cursor, CURSORA) |
 916                    FW_WM_VLV(wm->pipe[PIPE_A].sprite[0], SPRITEA));
 917         I915_WRITE(DSPFW3,
 918                    FW_WM(wm->sr.cursor, CURSOR_SR));
 919
 920         if (IS_CHERRYVIEW(dev_priv)) {
 921                 I915_WRITE(DSPFW7_CHV,
 922                            FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) |
 923                            FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
 924                 I915_WRITE(DSPFW8_CHV,
 925                            FW_WM_VLV(wm->pipe[PIPE_C].sprite[1], SPRITEF) |
 926                            FW_WM_VLV(wm->pipe[PIPE_C].sprite[0], SPRITEE));
 927                 I915_WRITE(DSPFW9_CHV,
 928                            FW_WM_VLV(wm->pipe[PIPE_C].primary, PLANEC) |
 929                            FW_WM(wm->pipe[PIPE_C].cursor, CURSORC));
 930                 I915_WRITE(DSPHOWM,
 931                            FW_WM(wm->sr.plane >> 9, SR_HI) |
 932                            FW_WM(wm->pipe[PIPE_C].sprite[1] >> 8, SPRITEF_HI) |
 933                            FW_WM(wm->pipe[PIPE_C].sprite[0] >> 8, SPRITEE_HI) |
 934                            FW_WM(wm->pipe[PIPE_C].primary >> 8, PLANEC_HI) |
 935                            FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) |
 936                            FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) |
 937                            FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) |
 938                            FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) |
 939                            FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) |
 940                            FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
 941         } else {
 942                 I915_WRITE(DSPFW7,
 943                            FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) |
 944                            FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
 945                 I915_WRITE(DSPHOWM,
 946                            FW_WM(wm->sr.plane >> 9, SR_HI) |
 947                            FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) |
 948                            FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) |
 949                            FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) |
 950                            FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) |
 951                            FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) |
 952                            FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
 953         }
 954
 955         /* zero (unused) WM1 watermarks */
 956         I915_WRITE(DSPFW4, 0);
 957         I915_WRITE(DSPFW5, 0);
 958         I915_WRITE(DSPFW6, 0);
 959         I915_WRITE(DSPHOWM1, 0);
 960
 961         POSTING_READ(DSPFW1);
 962 }
 963
 964 #undef FW_WM_VLV
 965
 966 enum vlv_wm_level {
 967         VLV_WM_LEVEL_PM2,
 968         VLV_WM_LEVEL_PM5,
 969         VLV_WM_LEVEL_DDR_DVFS,
 970 };
 971
 972 /* latency must be in 0.1us units. */
 973 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
 974                                    unsigned int pipe_htotal,
 975                                    unsigned int horiz_pixels,
 976                                    unsigned int bytes_per_pixel,
 977                                    unsigned int latency)
 978 {
 979         unsigned int ret;
 980
 981         ret = (latency * pixel_rate) / (pipe_htotal * 10000);
 982         ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
 983         ret = DIV_ROUND_UP(ret, 64);
 984
 985         return ret;
 986 }
 987
 988 static void vlv_setup_wm_latency(struct drm_device *dev)
 989 {
 990         struct drm_i915_private *dev_priv = dev->dev_private;
 991
 992         /* all latencies in usec */
 993         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
 994
 995         dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
 996
 997         if (IS_CHERRYVIEW(dev_priv)) {
 998                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
 999                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1000
1001                 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1002         }
1003 }
1004
1005 static uint16_t vlv_compute_wm_level(struct intel_plane *plane,
1006                                      struct intel_crtc *crtc,
1007                                      const struct intel_plane_state *state,
1008                                      int level)
1009 {
1010         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1011         int clock, htotal, pixel_size, width, wm;
1012
1013         if (dev_priv->wm.pri_latency[level] == 0)
1014                 return USHRT_MAX;
1015
1016         if (!state->visible)
1017                 return 0;
1018
1019         pixel_size = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
1020         clock = crtc->config->base.adjusted_mode.crtc_clock;
1021         htotal = crtc->config->base.adjusted_mode.crtc_htotal;
1022         width = crtc->config->pipe_src_w;
1023         if (WARN_ON(htotal == 0))
1024                 htotal = 1;
1025
1026         if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
1027                 /*
1028                  * FIXME the formula gives values that are
1029                  * too big for the cursor FIFO, and hence we
1030                  * would never be able to use cursors. For
1031                  * now just hardcode the watermark.
1032                  */
1033                 wm = 63;
1034         } else {
1035                 wm = vlv_wm_method2(clock, htotal, width, pixel_size,
1036                                     dev_priv->wm.pri_latency[level] * 10);
1037         }
1038
1039         return min_t(int, wm, USHRT_MAX);
1040 }
1041
1042 static void vlv_compute_fifo(struct intel_crtc *crtc)
1043 {
1044         struct drm_device *dev = crtc->base.dev;
1045         struct vlv_wm_state *wm_state = &crtc->wm_state;
1046         struct intel_plane *plane;
1047         unsigned int total_rate = 0;
1048         const int fifo_size = 512 - 1;
1049         int fifo_extra, fifo_left = fifo_size;
1050
1051         for_each_intel_plane_on_crtc(dev, crtc, plane) {
1052                 struct intel_plane_state *state =
1053                         to_intel_plane_state(plane->base.state);
1054
1055                 if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
1056                         continue;
1057
1058                 if (state->visible) {
1059                         wm_state->num_active_planes++;
1060                         total_rate += drm_format_plane_cpp(state->base.fb->pixel_format, 0);
1061                 }
1062         }
1063
1064         for_each_intel_plane_on_crtc(dev, crtc, plane) {
1065                 struct intel_plane_state *state =
1066                         to_intel_plane_state(plane->base.state);
1067                 unsigned int rate;
1068
1069                 if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
1070                         plane->wm.fifo_size = 63;
1071                         continue;
1072                 }
1073
1074                 if (!state->visible) {
1075                         plane->wm.fifo_size = 0;
1076                         continue;
1077                 }
1078
1079                 rate = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
1080                 plane->wm.fifo_size = fifo_size * rate / total_rate;
1081                 fifo_left -= plane->wm.fifo_size;
1082         }
1083
1084         fifo_extra = DIV_ROUND_UP(fifo_left, wm_state->num_active_planes ?: 1);
1085
1086         /* spread the remainder evenly */
1087         for_each_intel_plane_on_crtc(dev, crtc, plane) {
1088                 int plane_extra;
1089
1090                 if (fifo_left == 0)
1091                         break;
1092
1093                 if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
1094                         continue;
1095
1096                 /* give it all to the first plane if none are active */
1097                 if (plane->wm.fifo_size == 0 &&
1098                     wm_state->num_active_planes)
1099                         continue;
1100
1101                 plane_extra = min(fifo_extra, fifo_left);
1102                 plane->wm.fifo_size += plane_extra;
1103                 fifo_left -= plane_extra;
1104         }
1105
1106         WARN_ON(fifo_left != 0);
1107 }
1108
1109 static void vlv_invert_wms(struct intel_crtc *crtc)
1110 {
1111         struct vlv_wm_state *wm_state = &crtc->wm_state;
1112         int level;
1113
1114         for (level = 0; level < wm_state->num_levels; level++) {
1115                 struct drm_device *dev = crtc->base.dev;
1116                 const int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
1117                 struct intel_plane *plane;
1118
1119                 wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane;
1120                 wm_state->sr[level].cursor = 63 - wm_state->sr[level].cursor;
1121
1122                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
1123                         switch (plane->base.type) {
1124                                 int sprite;
1125                         case DRM_PLANE_TYPE_CURSOR:
1126                                 wm_state->wm[level].cursor = plane->wm.fifo_size -
1127                                         wm_state->wm[level].cursor;
1128                                 break;
1129                         case DRM_PLANE_TYPE_PRIMARY:
1130                                 wm_state->wm[level].primary = plane->wm.fifo_size -
1131                                         wm_state->wm[level].primary;
1132                                 break;
1133                         case DRM_PLANE_TYPE_OVERLAY:
1134                                 sprite = plane->plane;
1135                                 wm_state->wm[level].sprite[sprite] = plane->wm.fifo_size -
1136                                         wm_state->wm[level].sprite[sprite];
1137                                 break;
1138                         }
1139                 }
1140         }
1141 }
1142
1143 static void vlv_compute_wm(struct intel_crtc *crtc)
1144 {
1145         struct drm_device *dev = crtc->base.dev;
1146         struct vlv_wm_state *wm_state = &crtc->wm_state;
1147         struct intel_plane *plane;
1148         int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
1149         int level;
1150
1151         memset(wm_state, 0, sizeof(*wm_state));
1152
1153         wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed;
1154         wm_state->num_levels = to_i915(dev)->wm.max_level + 1;
1155
1156         wm_state->num_active_planes = 0;
1157
1158         vlv_compute_fifo(crtc);
1159
1160         if (wm_state->num_active_planes != 1)
1161                 wm_state->cxsr = false;
1162
1163         if (wm_state->cxsr) {
1164                 for (level = 0; level < wm_state->num_levels; level++) {
1165                         wm_state->sr[level].plane = sr_fifo_size;
1166                         wm_state->sr[level].cursor = 63;
1167                 }
1168         }
1169
1170         for_each_intel_plane_on_crtc(dev, crtc, plane) {
1171                 struct intel_plane_state *state =
1172                         to_intel_plane_state(plane->base.state);
1173
1174                 if (!state->visible)
1175                         continue;
1176
1177                 /* normal watermarks */
1178                 for (level = 0; level < wm_state->num_levels; level++) {
1179                         int wm = vlv_compute_wm_level(plane, crtc, state, level);
1180                         int max_wm = plane->base.type == DRM_PLANE_TYPE_CURSOR ? 63 : 511;
1181
1182                         /* hack */
1183                         if (WARN_ON(level == 0 && wm > max_wm))
1184                                 wm = max_wm;
1185
1186                         if (wm > plane->wm.fifo_size)
1187                                 break;
1188
1189                         switch (plane->base.type) {
1190                                 int sprite;
1191                         case DRM_PLANE_TYPE_CURSOR:
1192                                 wm_state->wm[level].cursor = wm;
1193                                 break;
1194                         case DRM_PLANE_TYPE_PRIMARY:
1195                                 wm_state->wm[level].primary = wm;
1196                                 break;
1197                         case DRM_PLANE_TYPE_OVERLAY:
1198                                 sprite = plane->plane;
1199                                 wm_state->wm[level].sprite[sprite] = wm;
1200                                 break;
1201                         }
1202                 }
1203
1204                 wm_state->num_levels = level;
1205
1206                 if (!wm_state->cxsr)
1207                         continue;
1208
1209                 /* maxfifo watermarks */
1210                 switch (plane->base.type) {
1211                         int sprite, level;
1212                 case DRM_PLANE_TYPE_CURSOR:
1213                         for (level = 0; level < wm_state->num_levels; level++)
1214                                 wm_state->sr[level].cursor =
1215                                         wm_state->sr[level].cursor;
1216                         break;
1217                 case DRM_PLANE_TYPE_PRIMARY:
1218                         for (level = 0; level < wm_state->num_levels; level++)
1219                                 wm_state->sr[level].plane =
1220                                         min(wm_state->sr[level].plane,
1221                                             wm_state->wm[level].primary);
1222                         break;
1223                 case DRM_PLANE_TYPE_OVERLAY:
1224                         sprite = plane->plane;
1225                         for (level = 0; level < wm_state->num_levels; level++)
1226                                 wm_state->sr[level].plane =
1227                                         min(wm_state->sr[level].plane,
1228                                             wm_state->wm[level].sprite[sprite]);
1229                         break;
1230                 }
1231         }
1232
1233         /* clear any (partially) filled invalid levels */
1234         for (level = wm_state->num_levels; level < to_i915(dev)->wm.max_level + 1; level++) {
1235                 memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level]));
1236                 memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level]));
1237         }
1238
1239         vlv_invert_wms(crtc);
1240 }
1241
1242 #define VLV_FIFO(plane, value) \
1243         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1244
1245 static void vlv_pipe_set_fifo_size(struct intel_crtc *crtc)
1246 {
1247         struct drm_device *dev = crtc->base.dev;
1248         struct drm_i915_private *dev_priv = to_i915(dev);
1249         struct intel_plane *plane;
1250         int sprite0_start = 0, sprite1_start = 0, fifo_size = 0;
1251
1252         for_each_intel_plane_on_crtc(dev, crtc, plane) {
1253                 if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
1254                         WARN_ON(plane->wm.fifo_size != 63);
1255                         continue;
1256                 }
1257
1258                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
1259                         sprite0_start = plane->wm.fifo_size;
1260                 else if (plane->plane == 0)
1261                         sprite1_start = sprite0_start + plane->wm.fifo_size;
1262                 else
1263                         fifo_size = sprite1_start + plane->wm.fifo_size;
1264         }
1265
1266         WARN_ON(fifo_size != 512 - 1);
1267
1268         DRM_DEBUG_KMS("Pipe %c FIFO split %d / %d / %d\n",
1269                       pipe_name(crtc->pipe), sprite0_start,
1270                       sprite1_start, fifo_size);
1271
1272         switch (crtc->pipe) {
1273                 uint32_t dsparb, dsparb2, dsparb3;
1274         case PIPE_A:
1275                 dsparb = I915_READ(DSPARB);
1276                 dsparb2 = I915_READ(DSPARB2);
1277
1278                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1279                             VLV_FIFO(SPRITEB, 0xff));
1280                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1281                            VLV_FIFO(SPRITEB, sprite1_start));
1282
1283                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1284                              VLV_FIFO(SPRITEB_HI, 0x1));
1285                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1286                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1287
1288                 I915_WRITE(DSPARB, dsparb);
1289                 I915_WRITE(DSPARB2, dsparb2);
1290                 break;
1291         case PIPE_B:
1292                 dsparb = I915_READ(DSPARB);
1293                 dsparb2 = I915_READ(DSPARB2);
1294
1295                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1296                             VLV_FIFO(SPRITED, 0xff));
1297                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1298                            VLV_FIFO(SPRITED, sprite1_start));
1299
1300                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1301                              VLV_FIFO(SPRITED_HI, 0xff));
1302                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1303                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1304
1305                 I915_WRITE(DSPARB, dsparb);
1306                 I915_WRITE(DSPARB2, dsparb2);
1307                 break;
1308         case PIPE_C:
1309                 dsparb3 = I915_READ(DSPARB3);
1310                 dsparb2 = I915_READ(DSPARB2);
1311
1312                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1313                              VLV_FIFO(SPRITEF, 0xff));
1314                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1315                             VLV_FIFO(SPRITEF, sprite1_start));
1316
1317                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1318                              VLV_FIFO(SPRITEF_HI, 0xff));
1319                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1320                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1321
1322                 I915_WRITE(DSPARB3, dsparb3);
1323                 I915_WRITE(DSPARB2, dsparb2);
1324                 break;
1325         default:
1326                 break;
1327         }
1328 }
1329
1330 #undef VLV_FIFO
1331
1332 static void vlv_merge_wm(struct drm_device *dev,
1333                          struct vlv_wm_values *wm)
1334 {
1335         struct intel_crtc *crtc;
1336         int num_active_crtcs = 0;
1337
1338         wm->level = to_i915(dev)->wm.max_level;
1339         wm->cxsr = true;
1340
1341         for_each_intel_crtc(dev, crtc) {
1342                 const struct vlv_wm_state *wm_state = &crtc->wm_state;
1343
1344                 if (!crtc->active)
1345                         continue;
1346
1347                 if (!wm_state->cxsr)
1348                         wm->cxsr = false;
1349
1350                 num_active_crtcs++;
1351                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
1352         }
1353
1354         if (num_active_crtcs != 1)
1355                 wm->cxsr = false;
1356
1357         if (num_active_crtcs > 1)
1358                 wm->level = VLV_WM_LEVEL_PM2;
1359
1360         for_each_intel_crtc(dev, crtc) {
1361                 struct vlv_wm_state *wm_state = &crtc->wm_state;
1362                 enum pipe pipe = crtc->pipe;
1363
1364                 if (!crtc->active)
1365                         continue;
1366
1367                 wm->pipe[pipe] = wm_state->wm[wm->level];
1368                 if (wm->cxsr)
1369                         wm->sr = wm_state->sr[wm->level];
1370
1371                 wm->ddl[pipe].primary = DDL_PRECISION_HIGH | 2;
1372                 wm->ddl[pipe].sprite[0] = DDL_PRECISION_HIGH | 2;
1373                 wm->ddl[pipe].sprite[1] = DDL_PRECISION_HIGH | 2;
1374                 wm->ddl[pipe].cursor = DDL_PRECISION_HIGH | 2;
1375         }
1376 }
1377
1378 static void vlv_update_wm(struct drm_crtc *crtc)
1379 {
1380         struct drm_device *dev = crtc->dev;
1381         struct drm_i915_private *dev_priv = dev->dev_private;
1382         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1383         enum pipe pipe = intel_crtc->pipe;
1384         struct vlv_wm_values wm = {};
1385
1386         vlv_compute_wm(intel_crtc);
1387         vlv_merge_wm(dev, &wm);
1388
1389         if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) {
1390                 /* FIXME should be part of crtc atomic commit */
1391                 vlv_pipe_set_fifo_size(intel_crtc);
1392                 return;
1393         }
1394
1395         if (wm.level < VLV_WM_LEVEL_DDR_DVFS &&
1396             dev_priv->wm.vlv.level >= VLV_WM_LEVEL_DDR_DVFS)
1397                 chv_set_memory_dvfs(dev_priv, false);
1398
1399         if (wm.level < VLV_WM_LEVEL_PM5 &&
1400             dev_priv->wm.vlv.level >= VLV_WM_LEVEL_PM5)
1401                 chv_set_memory_pm5(dev_priv, false);
1402
1403         if (!wm.cxsr && dev_priv->wm.vlv.cxsr)
1404                 intel_set_memory_cxsr(dev_priv, false);
1405
1406         /* FIXME should be part of crtc atomic commit */
1407         vlv_pipe_set_fifo_size(intel_crtc);
1408
1409         vlv_write_wm_values(intel_crtc, &wm);
1410
1411         DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, "
1412                       "sprite0=%d, sprite1=%d, SR: plane=%d, cursor=%d level=%d cxsr=%d\n",
1413                       pipe_name(pipe), wm.pipe[pipe].primary, wm.pipe[pipe].cursor,
1414                       wm.pipe[pipe].sprite[0], wm.pipe[pipe].sprite[1],
1415                       wm.sr.plane, wm.sr.cursor, wm.level, wm.cxsr);
1416
1417         if (wm.cxsr && !dev_priv->wm.vlv.cxsr)
1418                 intel_set_memory_cxsr(dev_priv, true);
1419
1420         if (wm.level >= VLV_WM_LEVEL_PM5 &&
1421             dev_priv->wm.vlv.level < VLV_WM_LEVEL_PM5)
1422                 chv_set_memory_pm5(dev_priv, true);
1423
1424         if (wm.level >= VLV_WM_LEVEL_DDR_DVFS &&
1425             dev_priv->wm.vlv.level < VLV_WM_LEVEL_DDR_DVFS)
1426                 chv_set_memory_dvfs(dev_priv, true);
1427
1428         dev_priv->wm.vlv = wm;
1429 }
1430
1431 #define single_plane_enabled(mask) is_power_of_2(mask)
1432
1433 static void g4x_update_wm(struct drm_crtc *crtc)
1434 {
1435         struct drm_device *dev = crtc->dev;
1436         static const int sr_latency_ns = 12000;
1437         struct drm_i915_private *dev_priv = dev->dev_private;
1438         int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1439         int plane_sr, cursor_sr;
1440         unsigned int enabled = 0;
1441         bool cxsr_enabled;
1442
1443         if (g4x_compute_wm0(dev, PIPE_A,
1444                             &g4x_wm_info, pessimal_latency_ns,
1445                             &g4x_cursor_wm_info, pessimal_latency_ns,
1446                             &planea_wm, &cursora_wm))
1447                 enabled |= 1 << PIPE_A;
1448
1449         if (g4x_compute_wm0(dev, PIPE_B,
1450                             &g4x_wm_info, pessimal_latency_ns,
1451                             &g4x_cursor_wm_info, pessimal_latency_ns,
1452                             &planeb_wm, &cursorb_wm))
1453                 enabled |= 1 << PIPE_B;
1454
1455         if (single_plane_enabled(enabled) &&
1456             g4x_compute_srwm(dev, ffs(enabled) - 1,
1457                              sr_latency_ns,
1458                              &g4x_wm_info,
1459                              &g4x_cursor_wm_info,
1460                              &plane_sr, &cursor_sr)) {
1461                 cxsr_enabled = true;
1462         } else {
1463                 cxsr_enabled = false;
1464                 intel_set_memory_cxsr(dev_priv, false);
1465                 plane_sr = cursor_sr = 0;
1466         }
1467
1468         DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
1469                       "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1470                       planea_wm, cursora_wm,
1471                       planeb_wm, cursorb_wm,
1472                       plane_sr, cursor_sr);
1473
1474         I915_WRITE(DSPFW1,
1475                    FW_WM(plane_sr, SR) |
1476                    FW_WM(cursorb_wm, CURSORB) |
1477                    FW_WM(planeb_wm, PLANEB) |
1478                    FW_WM(planea_wm, PLANEA));
1479         I915_WRITE(DSPFW2,
1480                    (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
1481                    FW_WM(cursora_wm, CURSORA));
1482         /* HPLL off in SR has some issues on G4x... disable it */
1483         I915_WRITE(DSPFW3,
1484                    (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
1485                    FW_WM(cursor_sr, CURSOR_SR));
1486
1487         if (cxsr_enabled)
1488                 intel_set_memory_cxsr(dev_priv, true);
1489 }
1490
1491 static void i965_update_wm(struct drm_crtc *unused_crtc)
1492 {
1493         struct drm_device *dev = unused_crtc->dev;
1494         struct drm_i915_private *dev_priv = dev->dev_private;
1495         struct drm_crtc *crtc;
1496         int srwm = 1;
1497         int cursor_sr = 16;
1498         bool cxsr_enabled;
1499
1500         /* Calc sr entries for one plane configs */
1501         crtc = single_enabled_crtc(dev);
1502         if (crtc) {
1503                 /* self-refresh has much higher latency */
1504                 static const int sr_latency_ns = 12000;
1505                 const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1506                 int clock = adjusted_mode->crtc_clock;
1507                 int htotal = adjusted_mode->crtc_htotal;
1508                 int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
1509                 int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
1510                 unsigned long line_time_us;
1511                 int entries;
1512
1513                 line_time_us = max(htotal * 1000 / clock, 1);
1514
1515                 /* Use ns/us then divide to preserve precision */
1516                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1517                         pixel_size * hdisplay;
1518                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
1519                 srwm = I965_FIFO_SIZE - entries;
1520                 if (srwm < 0)
1521                         srwm = 1;
1522                 srwm &= 0x1ff;
1523                 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1524                               entries, srwm);
1525
1526                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1527                         pixel_size * crtc->cursor->state->crtc_w;
1528                 entries = DIV_ROUND_UP(entries,
1529                                           i965_cursor_wm_info.cacheline_size);
1530                 cursor_sr = i965_cursor_wm_info.fifo_size -
1531                         (entries + i965_cursor_wm_info.guard_size);
1532
1533                 if (cursor_sr > i965_cursor_wm_info.max_wm)
1534                         cursor_sr = i965_cursor_wm_info.max_wm;
1535
1536                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1537                               "cursor %d\n", srwm, cursor_sr);
1538
1539                 cxsr_enabled = true;
1540         } else {
1541                 cxsr_enabled = false;
1542                 /* Turn off self refresh if both pipes are enabled */
1543                 intel_set_memory_cxsr(dev_priv, false);
1544         }
1545
1546         DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1547                       srwm);
1548
1549         /* 965 has limitations... */
1550         I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
1551                    FW_WM(8, CURSORB) |
1552                    FW_WM(8, PLANEB) |
1553                    FW_WM(8, PLANEA));
1554         I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
1555                    FW_WM(8, PLANEC_OLD));
1556         /* update cursor SR watermark */
1557         I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
1558
1559         if (cxsr_enabled)
1560                 intel_set_memory_cxsr(dev_priv, true);
1561 }
1562
1563 #undef FW_WM
1564
1565 static void i9xx_update_wm(struct drm_crtc *unused_crtc)
1566 {
1567         struct drm_device *dev = unused_crtc->dev;
1568         struct drm_i915_private *dev_priv = dev->dev_private;
1569         const struct intel_watermark_params *wm_info;
1570         uint32_t fwater_lo;
1571         uint32_t fwater_hi;
1572         int cwm, srwm = 1;
1573         int fifo_size;
1574         int planea_wm, planeb_wm;
1575         struct drm_crtc *crtc, *enabled = NULL;
1576
1577         if (IS_I945GM(dev))
1578                 wm_info = &i945_wm_info;
1579         else if (!IS_GEN2(dev))
1580                 wm_info = &i915_wm_info;
1581         else
1582                 wm_info = &i830_a_wm_info;
1583
1584         fifo_size = dev_priv->display.get_fifo_size(dev, 0);
1585         crtc = intel_get_crtc_for_plane(dev, 0);
1586         if (intel_crtc_active(crtc)) {
1587                 const struct drm_display_mode *adjusted_mode;
1588                 int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
1589                 if (IS_GEN2(dev))
1590                         cpp = 4;
1591
1592                 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1593                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1594                                                wm_info, fifo_size, cpp,
1595                                                pessimal_latency_ns);
1596                 enabled = crtc;
1597         } else {
1598                 planea_wm = fifo_size - wm_info->guard_size;
1599                 if (planea_wm > (long)wm_info->max_wm)
1600                         planea_wm = wm_info->max_wm;
1601         }
1602
1603         if (IS_GEN2(dev))
1604                 wm_info = &i830_bc_wm_info;
1605
1606         fifo_size = dev_priv->display.get_fifo_size(dev, 1);
1607         crtc = intel_get_crtc_for_plane(dev, 1);
1608         if (intel_crtc_active(crtc)) {
1609                 const struct drm_display_mode *adjusted_mode;
1610                 int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
1611                 if (IS_GEN2(dev))
1612                         cpp = 4;
1613
1614                 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1615                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1616                                                wm_info, fifo_size, cpp,
1617                                                pessimal_latency_ns);
1618                 if (enabled == NULL)
1619                         enabled = crtc;
1620                 else
1621                         enabled = NULL;
1622         } else {
1623                 planeb_wm = fifo_size - wm_info->guard_size;
1624                 if (planeb_wm > (long)wm_info->max_wm)
1625                         planeb_wm = wm_info->max_wm;
1626         }
1627
1628         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
1629
1630         if (IS_I915GM(dev) && enabled) {
1631                 struct drm_i915_gem_object *obj;
1632
1633                 obj = intel_fb_obj(enabled->primary->state->fb);
1634
1635                 /* self-refresh seems busted with untiled */
1636                 if (obj->tiling_mode == I915_TILING_NONE)
1637                         enabled = NULL;
1638         }
1639
1640         /*
1641          * Overlay gets an aggressive default since video jitter is bad.
1642          */
1643         cwm = 2;
1644
1645         /* Play safe and disable self-refresh before adjusting watermarks. */
1646         intel_set_memory_cxsr(dev_priv, false);
1647
1648         /* Calc sr entries for one plane configs */
1649         if (HAS_FW_BLC(dev) && enabled) {
1650                 /* self-refresh has much higher latency */
1651                 static const int sr_latency_ns = 6000;
1652                 const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode;
1653                 int clock = adjusted_mode->crtc_clock;
1654                 int htotal = adjusted_mode->crtc_htotal;
1655                 int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
1656                 int pixel_size = enabled->primary->state->fb->bits_per_pixel / 8;
1657                 unsigned long line_time_us;
1658                 int entries;
1659
1660                 line_time_us = max(htotal * 1000 / clock, 1);
1661
1662                 /* Use ns/us then divide to preserve precision */
1663                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1664                         pixel_size * hdisplay;
1665                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
1666                 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
1667                 srwm = wm_info->fifo_size - entries;
1668                 if (srwm < 0)
1669                         srwm = 1;
1670
1671                 if (IS_I945G(dev) || IS_I945GM(dev))
1672                         I915_WRITE(FW_BLC_SELF,
1673                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
1674                 else if (IS_I915GM(dev))
1675                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
1676         }
1677
1678         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1679                       planea_wm, planeb_wm, cwm, srwm);
1680
1681         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
1682         fwater_hi = (cwm & 0x1f);
1683
1684         /* Set request length to 8 cachelines per fetch */
1685         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
1686         fwater_hi = fwater_hi | (1 << 8);
1687
1688         I915_WRITE(FW_BLC, fwater_lo);
1689         I915_WRITE(FW_BLC2, fwater_hi);
1690
1691         if (enabled)
1692                 intel_set_memory_cxsr(dev_priv, true);
1693 }
1694
1695 static void i845_update_wm(struct drm_crtc *unused_crtc)
1696 {
1697         struct drm_device *dev = unused_crtc->dev;
1698         struct drm_i915_private *dev_priv = dev->dev_private;
1699         struct drm_crtc *crtc;
1700         const struct drm_display_mode *adjusted_mode;
1701         uint32_t fwater_lo;
1702         int planea_wm;
1703
1704         crtc = single_enabled_crtc(dev);
1705         if (crtc == NULL)
1706                 return;
1707
1708         adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1709         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1710                                        &i845_wm_info,
1711                                        dev_priv->display.get_fifo_size(dev, 0),
1712                                        4, pessimal_latency_ns);
1713         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
1714         fwater_lo |= (3<<8) | planea_wm;
1715
1716         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
1717
1718         I915_WRITE(FW_BLC, fwater_lo);
1719 }
1720
1721 uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
1722 {
1723         uint32_t pixel_rate;
1724
1725         pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
1726
1727         /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
1728          * adjust the pixel_rate here. */
1729
1730         if (pipe_config->pch_pfit.enabled) {
1731                 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
1732                 uint32_t pfit_size = pipe_config->pch_pfit.size;
1733
1734                 pipe_w = pipe_config->pipe_src_w;
1735                 pipe_h = pipe_config->pipe_src_h;
1736
1737                 pfit_w = (pfit_size >> 16) & 0xFFFF;
1738                 pfit_h = pfit_size & 0xFFFF;
1739                 if (pipe_w < pfit_w)
1740                         pipe_w = pfit_w;
1741                 if (pipe_h < pfit_h)
1742                         pipe_h = pfit_h;
1743
1744                 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
1745                                      pfit_w * pfit_h);
1746         }
1747
1748         return pixel_rate;
1749 }
1750
1751 /* latency must be in 0.1us units. */
1752 static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
1753                                uint32_t latency)
1754 {
1755         uint64_t ret;
1756
1757         if (WARN(latency == 0, "Latency value missing\n"))
1758                 return UINT_MAX;
1759
1760         ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
1761         ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
1762
1763         return ret;
1764 }
1765
1766 /* latency must be in 0.1us units. */
1767 static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
1768                                uint32_t horiz_pixels, uint8_t bytes_per_pixel,
1769                                uint32_t latency)
1770 {
1771         uint32_t ret;
1772
1773         if (WARN(latency == 0, "Latency value missing\n"))
1774                 return UINT_MAX;
1775
1776         ret = (latency * pixel_rate) / (pipe_htotal * 10000);
1777         ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
1778         ret = DIV_ROUND_UP(ret, 64) + 2;
1779         return ret;
1780 }
1781
1782 static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
1783                            uint8_t bytes_per_pixel)
1784 {
1785         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
1786 }
1787
1788 struct skl_pipe_wm_parameters {
1789         bool active;
1790         uint32_t pipe_htotal;
1791         uint32_t pixel_rate; /* in KHz */
1792         struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
1793         struct intel_plane_wm_parameters cursor;
1794 };
1795
1796 struct ilk_pipe_wm_parameters {
1797         bool active;
1798         uint32_t pipe_htotal;
1799         uint32_t pixel_rate;
1800         struct intel_plane_wm_parameters pri;
1801         struct intel_plane_wm_parameters spr;
1802         struct intel_plane_wm_parameters cur;
1803 };
1804
1805 struct ilk_wm_maximums {
1806         uint16_t pri;
1807         uint16_t spr;
1808         uint16_t cur;
1809         uint16_t fbc;
1810 };
1811
1812 /* used in computing the new watermarks state */
1813 struct intel_wm_config {
1814         unsigned int num_pipes_active;
1815         bool sprites_enabled;
1816         bool sprites_scaled;
1817 };
1818
1819 /*
1820  * For both WM_PIPE and WM_LP.
1821  * mem_value must be in 0.1us units.
1822  */
1823 static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
1824                                    uint32_t mem_value,
1825                                    bool is_lp)
1826 {
1827         uint32_t method1, method2;
1828
1829         if (!params->active || !params->pri.enabled)
1830                 return 0;
1831
1832         method1 = ilk_wm_method1(params->pixel_rate,
1833                                  params->pri.bytes_per_pixel,
1834                                  mem_value);
1835
1836         if (!is_lp)
1837                 return method1;
1838
1839         method2 = ilk_wm_method2(params->pixel_rate,
1840                                  params->pipe_htotal,
1841                                  params->pri.horiz_pixels,
1842                                  params->pri.bytes_per_pixel,
1843                                  mem_value);
1844
1845         return min(method1, method2);
1846 }
1847
1848 /*
1849  * For both WM_PIPE and WM_LP.
1850  * mem_value must be in 0.1us units.
1851  */
1852 static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
1853                                    uint32_t mem_value)
1854 {
1855         uint32_t method1, method2;
1856
1857         if (!params->active || !params->spr.enabled)
1858                 return 0;
1859
1860         method1 = ilk_wm_method1(params->pixel_rate,
1861                                  params->spr.bytes_per_pixel,
1862                                  mem_value);
1863         method2 = ilk_wm_method2(params->pixel_rate,
1864                                  params->pipe_htotal,
1865                                  params->spr.horiz_pixels,
1866                                  params->spr.bytes_per_pixel,
1867                                  mem_value);
1868         return min(method1, method2);
1869 }
1870
1871 /*
1872  * For both WM_PIPE and WM_LP.
1873  * mem_value must be in 0.1us units.
1874  */
1875 static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
1876                                    uint32_t mem_value)
1877 {
1878         if (!params->active || !params->cur.enabled)
1879                 return 0;
1880
1881         return ilk_wm_method2(params->pixel_rate,
1882                               params->pipe_htotal,
1883                               params->cur.horiz_pixels,
1884                               params->cur.bytes_per_pixel,
1885                               mem_value);
1886 }
1887
1888 /* Only for WM_LP. */
1889 static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
1890                                    uint32_t pri_val)
1891 {
1892         if (!params->active || !params->pri.enabled)
1893                 return 0;
1894
1895         return ilk_wm_fbc(pri_val,
1896                           params->pri.horiz_pixels,
1897                           params->pri.bytes_per_pixel);
1898 }
1899
1900 static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
1901 {
1902         if (INTEL_INFO(dev)->gen >= 8)
1903                 return 3072;
1904         else if (INTEL_INFO(dev)->gen >= 7)
1905                 return 768;
1906         else
1907                 return 512;
1908 }
1909
1910 static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
1911                                          int level, bool is_sprite)
1912 {
1913         if (INTEL_INFO(dev)->gen >= 8)
1914                 /* BDW primary/sprite plane watermarks */
1915                 return level == 0 ? 255 : 2047;
1916         else if (INTEL_INFO(dev)->gen >= 7)
1917                 /* IVB/HSW primary/sprite plane watermarks */
1918                 return level == 0 ? 127 : 1023;
1919         else if (!is_sprite)
1920                 /* ILK/SNB primary plane watermarks */
1921                 return level == 0 ? 127 : 511;
1922         else
1923                 /* ILK/SNB sprite plane watermarks */
1924                 return level == 0 ? 63 : 255;
1925 }
1926
1927 static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
1928                                           int level)
1929 {
1930         if (INTEL_INFO(dev)->gen >= 7)
1931                 return level == 0 ? 63 : 255;
1932         else
1933                 return level == 0 ? 31 : 63;
1934 }
1935
1936 static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
1937 {
1938         if (INTEL_INFO(dev)->gen >= 8)
1939                 return 31;
1940         else
1941                 return 15;
1942 }
1943
1944 /* Calculate the maximum primary/sprite plane watermark */
1945 static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
1946                                      int level,
1947                                      const struct intel_wm_config *config,
1948                                      enum intel_ddb_partitioning ddb_partitioning,
1949                                      bool is_sprite)
1950 {
1951         unsigned int fifo_size = ilk_display_fifo_size(dev);
1952
1953         /* if sprites aren't enabled, sprites get nothing */
1954         if (is_sprite && !config->sprites_enabled)
1955                 return 0;
1956
1957         /* HSW allows LP1+ watermarks even with multiple pipes */
1958         if (level == 0 || config->num_pipes_active > 1) {
1959                 fifo_size /= INTEL_INFO(dev)->num_pipes;
1960
1961                 /*
1962                  * For some reason the non self refresh
1963                  * FIFO size is only half of the self
1964                  * refresh FIFO size on ILK/SNB.
1965                  */
1966                 if (INTEL_INFO(dev)->gen <= 6)
1967                         fifo_size /= 2;
1968         }
1969
1970         if (config->sprites_enabled) {
1971                 /* level 0 is always calculated with 1:1 split */
1972                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
1973                         if (is_sprite)
1974                                 fifo_size *= 5;
1975                         fifo_size /= 6;
1976                 } else {
1977                         fifo_size /= 2;
1978                 }
1979         }
1980
1981         /* clamp to max that the registers can hold */
1982         return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
1983 }
1984
1985 /* Calculate the maximum cursor plane watermark */
1986 static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
1987                                       int level,
1988                                       const struct intel_wm_config *config)
1989 {
1990         /* HSW LP1+ watermarks w/ multiple pipes */
1991         if (level > 0 && config->num_pipes_active > 1)
1992                 return 64;
1993
1994         /* otherwise just report max that registers can hold */
1995         return ilk_cursor_wm_reg_max(dev, level);
1996 }
1997
1998 static void ilk_compute_wm_maximums(const struct drm_device *dev,
1999                                     int level,
2000                                     const struct intel_wm_config *config,
2001                                     enum intel_ddb_partitioning ddb_partitioning,
2002                                     struct ilk_wm_maximums *max)
2003 {
2004         max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
2005         max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
2006         max->cur = ilk_cursor_wm_max(dev, level, config);
2007         max->fbc = ilk_fbc_wm_reg_max(dev);
2008 }
2009
2010 static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
2011                                         int level,
2012                                         struct ilk_wm_maximums *max)
2013 {
2014         max->pri = ilk_plane_wm_reg_max(dev, level, false);
2015         max->spr = ilk_plane_wm_reg_max(dev, level, true);
2016         max->cur = ilk_cursor_wm_reg_max(dev, level);
2017         max->fbc = ilk_fbc_wm_reg_max(dev);
2018 }
2019
2020 static bool ilk_validate_wm_level(int level,
2021                                   const struct ilk_wm_maximums *max,
2022                                   struct intel_wm_level *result)
2023 {
2024         bool ret;
2025
2026         /* already determined to be invalid? */
2027         if (!result->enable)
2028                 return false;
2029
2030         result->enable = result->pri_val <= max->pri &&
2031                          result->spr_val <= max->spr &&
2032                          result->cur_val <= max->cur;
2033
2034         ret = result->enable;
2035
2036         /*
2037          * HACK until we can pre-compute everything,
2038          * and thus fail gracefully if LP0 watermarks
2039          * are exceeded...
2040          */
2041         if (level == 0 && !result->enable) {
2042                 if (result->pri_val > max->pri)
2043                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2044                                       level, result->pri_val, max->pri);
2045                 if (result->spr_val > max->spr)
2046                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2047                                       level, result->spr_val, max->spr);
2048                 if (result->cur_val > max->cur)
2049                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2050                                       level, result->cur_val, max->cur);
2051
2052                 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
2053                 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
2054                 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
2055                 result->enable = true;
2056         }
2057
2058         return ret;
2059 }
2060
2061 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2062                                  int level,
2063                                  const struct ilk_pipe_wm_parameters *p,
2064                                  struct intel_wm_level *result)
2065 {
2066         uint16_t pri_latency = dev_priv->wm.pri_latency[level];
2067         uint16_t spr_latency = dev_priv->wm.spr_latency[level];
2068         uint16_t cur_latency = dev_priv->wm.cur_latency[level];
2069
2070         /* WM1+ latency values stored in 0.5us units */
2071         if (level > 0) {
2072                 pri_latency *= 5;
2073                 spr_latency *= 5;
2074                 cur_latency *= 5;
2075         }
2076
2077         result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
2078         result->spr_val = ilk_compute_spr_wm(p, spr_latency);
2079         result->cur_val = ilk_compute_cur_wm(p, cur_latency);
2080         result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
2081         result->enable = true;
2082 }
2083
2084 static uint32_t
2085 hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc)
2086 {
2087         struct drm_i915_private *dev_priv = dev->dev_private;
2088         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2089         struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
2090         u32 linetime, ips_linetime;
2091
2092         if (!intel_crtc->active)
2093                 return 0;
2094
2095         /* The WM are computed with base on how long it takes to fill a single
2096          * row at the given clock rate, multiplied by 8.
2097          * */
2098         linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2099                                      adjusted_mode->crtc_clock);
2100         ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2101                                          dev_priv->cdclk_freq);
2102
2103         return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2104                PIPE_WM_LINETIME_TIME(linetime);
2105 }
2106
2107 static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
2108 {
2109         struct drm_i915_private *dev_priv = dev->dev_private;
2110
2111         if (IS_GEN9(dev)) {
2112                 uint32_t val;
2113                 int ret, i;
2114                 int level, max_level = ilk_wm_max_level(dev);
2115
2116                 /* read the first set of memory latencies[0:3] */
2117                 val = 0; /* data0 to be programmed to 0 for first set */
2118                 mutex_lock(&dev_priv->rps.hw_lock);
2119                 ret = sandybridge_pcode_read(dev_priv,
2120                                              GEN9_PCODE_READ_MEM_LATENCY,
2121                                              &val);
2122                 mutex_unlock(&dev_priv->rps.hw_lock);
2123
2124                 if (ret) {
2125                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2126                         return;
2127                 }
2128
2129                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2130                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2131                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2132                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2133                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2134                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2135                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2136
2137                 /* read the second set of memory latencies[4:7] */
2138                 val = 1; /* data0 to be programmed to 1 for second set */
2139                 mutex_lock(&dev_priv->rps.hw_lock);
2140                 ret = sandybridge_pcode_read(dev_priv,
2141                                              GEN9_PCODE_READ_MEM_LATENCY,
2142                                              &val);
2143                 mutex_unlock(&dev_priv->rps.hw_lock);
2144                 if (ret) {
2145                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2146                         return;
2147                 }
2148
2149                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2150                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2151                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2152                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2153                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2154                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2155                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2156
2157                 /*
2158                  * WaWmMemoryReadLatency:skl
2159                  *
2160                  * punit doesn't take into account the read latency so we need
2161                  * to add 2us to the various latency levels we retrieve from
2162                  * the punit.
2163                  *   - W0 is a bit special in that it's the only level that
2164                  *   can't be disabled if we want to have display working, so
2165                  *   we always add 2us there.
2166                  *   - For levels >=1, punit returns 0us latency when they are
2167                  *   disabled, so we respect that and don't add 2us then
2168                  *
2169                  * Additionally, if a level n (n > 1) has a 0us latency, all
2170                  * levels m (m >= n) need to be disabled. We make sure to
2171                  * sanitize the values out of the punit to satisfy this
2172                  * requirement.
2173                  */
2174                 wm[0] += 2;
2175                 for (level = 1; level <= max_level; level++)
2176                         if (wm[level] != 0)
2177                                 wm[level] += 2;
2178                         else {
2179                                 for (i = level + 1; i <= max_level; i++)
2180                                         wm[i] = 0;
2181
2182                                 break;
2183                         }
2184         } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2185                 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2186
2187                 wm[0] = (sskpd >> 56) & 0xFF;
2188                 if (wm[0] == 0)
2189                         wm[0] = sskpd & 0xF;
2190                 wm[1] = (sskpd >> 4) & 0xFF;
2191                 wm[2] = (sskpd >> 12) & 0xFF;
2192                 wm[3] = (sskpd >> 20) & 0x1FF;
2193                 wm[4] = (sskpd >> 32) & 0x1FF;
2194         } else if (INTEL_INFO(dev)->gen >= 6) {
2195                 uint32_t sskpd = I915_READ(MCH_SSKPD);
2196
2197                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2198                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2199                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2200                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2201         } else if (INTEL_INFO(dev)->gen >= 5) {
2202                 uint32_t mltr = I915_READ(MLTR_ILK);
2203
2204                 /* ILK primary LP0 latency is 700 ns */
2205                 wm[0] = 7;
2206                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2207                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2208         }
2209 }
2210
2211 static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
2212 {
2213         /* ILK sprite LP0 latency is 1300 ns */
2214         if (INTEL_INFO(dev)->gen == 5)
2215                 wm[0] = 13;
2216 }
2217
2218 static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
2219 {
2220         /* ILK cursor LP0 latency is 1300 ns */
2221         if (INTEL_INFO(dev)->gen == 5)
2222                 wm[0] = 13;
2223
2224         /* WaDoubleCursorLP3Latency:ivb */
2225         if (IS_IVYBRIDGE(dev))
2226                 wm[3] *= 2;
2227 }
2228
2229 int ilk_wm_max_level(const struct drm_device *dev)
2230 {
2231         /* how many WM levels are we expecting */
2232         if (INTEL_INFO(dev)->gen >= 9)
2233                 return 7;
2234         else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2235                 return 4;
2236         else if (INTEL_INFO(dev)->gen >= 6)
2237                 return 3;
2238         else
2239                 return 2;
2240 }
2241
2242 static void intel_print_wm_latency(struct drm_device *dev,
2243                                    const char *name,
2244                                    const uint16_t wm[8])
2245 {
2246         int level, max_level = ilk_wm_max_level(dev);
2247
2248         for (level = 0; level <= max_level; level++) {
2249                 unsigned int latency = wm[level];
2250
2251                 if (latency == 0) {
2252                         DRM_ERROR("%s WM%d latency not provided\n",
2253                                   name, level);
2254                         continue;
2255                 }
2256
2257                 /*
2258                  * - latencies are in us on gen9.
2259                  * - before then, WM1+ latency values are in 0.5us units
2260                  */
2261                 if (IS_GEN9(dev))
2262                         latency *= 10;
2263                 else if (level > 0)
2264                         latency *= 5;
2265
2266                 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2267                               name, level, wm[level],
2268                               latency / 10, latency % 10);
2269         }
2270 }
2271
2272 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2273                                     uint16_t wm[5], uint16_t min)
2274 {
2275         int level, max_level = ilk_wm_max_level(dev_priv->dev);
2276
2277         if (wm[0] >= min)
2278                 return false;
2279
2280         wm[0] = max(wm[0], min);
2281         for (level = 1; level <= max_level; level++)
2282                 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
2283
2284         return true;
2285 }
2286
2287 static void snb_wm_latency_quirk(struct drm_device *dev)
2288 {
2289         struct drm_i915_private *dev_priv = dev->dev_private;
2290         bool changed;
2291
2292         /*
2293          * The BIOS provided WM memory latency values are often
2294          * inadequate for high resolution displays. Adjust them.
2295          */
2296         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
2297                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
2298                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
2299
2300         if (!changed)
2301                 return;
2302
2303         DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
2304         intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
2305         intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2306         intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
2307 }
2308
2309 static void ilk_setup_wm_latency(struct drm_device *dev)
2310 {
2311         struct drm_i915_private *dev_priv = dev->dev_private;
2312
2313         intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
2314
2315         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
2316                sizeof(dev_priv->wm.pri_latency));
2317         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
2318                sizeof(dev_priv->wm.pri_latency));
2319
2320         intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
2321         intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
2322
2323         intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
2324         intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2325         intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
2326
2327         if (IS_GEN6(dev))
2328                 snb_wm_latency_quirk(dev);
2329 }
2330
2331 static void skl_setup_wm_latency(struct drm_device *dev)
2332 {
2333         struct drm_i915_private *dev_priv = dev->dev_private;
2334
2335         intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
2336         intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
2337 }
2338
2339 static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
2340                                       struct ilk_pipe_wm_parameters *p)
2341 {
2342         struct drm_device *dev = crtc->dev;
2343         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2344         enum pipe pipe = intel_crtc->pipe;
2345         struct drm_plane *plane;
2346
2347         if (!intel_crtc->active)
2348                 return;
2349
2350         p->active = true;
2351         p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
2352         p->pixel_rate = ilk_pipe_pixel_rate(intel_crtc->config);
2353
2354         if (crtc->primary->state->fb)
2355                 p->pri.bytes_per_pixel =
2356                         crtc->primary->state->fb->bits_per_pixel / 8;
2357         else
2358                 p->pri.bytes_per_pixel = 4;
2359
2360         p->cur.bytes_per_pixel = 4;
2361         /*
2362          * TODO: for now, assume primary and cursor planes are always enabled.
2363          * Setting them to false makes the screen flicker.
2364          */
2365         p->pri.enabled = true;
2366         p->cur.enabled = true;
2367
2368         p->pri.horiz_pixels = intel_crtc->config->pipe_src_w;
2369         p->cur.horiz_pixels = intel_crtc->base.cursor->state->crtc_w;
2370
2371         drm_for_each_legacy_plane(plane, dev) {
2372                 struct intel_plane *intel_plane = to_intel_plane(plane);
2373
2374                 if (intel_plane->pipe == pipe) {
2375                         p->spr = intel_plane->wm;
2376                         break;
2377                 }
2378         }
2379 }
2380
2381 static void ilk_compute_wm_config(struct drm_device *dev,
2382                                   struct intel_wm_config *config)
2383 {
2384         struct intel_crtc *intel_crtc;
2385
2386         /* Compute the currently _active_ config */
2387         for_each_intel_crtc(dev, intel_crtc) {
2388                 const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
2389
2390                 if (!wm->pipe_enabled)
2391                         continue;
2392
2393                 config->sprites_enabled |= wm->sprites_enabled;
2394                 config->sprites_scaled |= wm->sprites_scaled;
2395                 config->num_pipes_active++;
2396         }
2397 }
2398
2399 /* Compute new watermarks for the pipe */
2400 static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
2401                                   const struct ilk_pipe_wm_parameters *params,
2402                                   struct intel_pipe_wm *pipe_wm)
2403 {
2404         struct drm_device *dev = crtc->dev;
2405         const struct drm_i915_private *dev_priv = dev->dev_private;
2406         int level, max_level = ilk_wm_max_level(dev);
2407         /* LP0 watermark maximums depend on this pipe alone */
2408         struct intel_wm_config config = {
2409                 .num_pipes_active = 1,
2410                 .sprites_enabled = params->spr.enabled,
2411                 .sprites_scaled = params->spr.scaled,
2412         };
2413         struct ilk_wm_maximums max;
2414
2415         pipe_wm->pipe_enabled = params->active;
2416         pipe_wm->sprites_enabled = params->spr.enabled;
2417         pipe_wm->sprites_scaled = params->spr.scaled;
2418
2419         /* ILK/SNB: LP2+ watermarks only w/o sprites */
2420         if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
2421                 max_level = 1;
2422
2423         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2424         if (params->spr.scaled)
2425                 max_level = 0;
2426
2427         ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]);
2428
2429         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2430                 pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
2431
2432         /* LP0 watermarks always use 1/2 DDB partitioning */
2433         ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
2434
2435         /* At least LP0 must be valid */
2436         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
2437                 return false;
2438
2439         ilk_compute_wm_reg_maximums(dev, 1, &max);
2440
2441         for (level = 1; level <= max_level; level++) {
2442                 struct intel_wm_level wm = {};
2443
2444                 ilk_compute_wm_level(dev_priv, level, params, &wm);
2445
2446                 /*
2447                  * Disable any watermark level that exceeds the
2448                  * register maximums since such watermarks are
2449                  * always invalid.
2450                  */
2451                 if (!ilk_validate_wm_level(level, &max, &wm))
2452                         break;
2453
2454                 pipe_wm->wm[level] = wm;
2455         }
2456
2457         return true;
2458 }
2459
2460 /*
2461  * Merge the watermarks from all active pipes for a specific level.
2462  */
2463 static void ilk_merge_wm_level(struct drm_device *dev,
2464                                int level,
2465                                struct intel_wm_level *ret_wm)
2466 {
2467         const struct intel_crtc *intel_crtc;
2468
2469         ret_wm->enable = true;
2470
2471         for_each_intel_crtc(dev, intel_crtc) {
2472                 const struct intel_pipe_wm *active = &intel_crtc->wm.active;
2473                 const struct intel_wm_level *wm = &active->wm[level];
2474
2475                 if (!active->pipe_enabled)
2476                         continue;
2477
2478                 /*
2479                  * The watermark values may have been used in the past,
2480                  * so we must maintain them in the registers for some
2481                  * time even if the level is now disabled.
2482                  */
2483                 if (!wm->enable)
2484                         ret_wm->enable = false;
2485
2486                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2487                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2488                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2489                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2490         }
2491 }
2492
2493 /*
2494  * Merge all low power watermarks for all active pipes.
2495  */
2496 static void ilk_wm_merge(struct drm_device *dev,
2497                          const struct intel_wm_config *config,
2498                          const struct ilk_wm_maximums *max,
2499                          struct intel_pipe_wm *merged)
2500 {
2501         struct drm_i915_private *dev_priv = dev->dev_private;
2502         int level, max_level = ilk_wm_max_level(dev);
2503         int last_enabled_level = max_level;
2504
2505         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
2506         if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
2507             config->num_pipes_active > 1)
2508                 return;
2509
2510         /* ILK: FBC WM must be disabled always */
2511         merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
2512
2513         /* merge each WM1+ level */
2514         for (level = 1; level <= max_level; level++) {
2515                 struct intel_wm_level *wm = &merged->wm[level];
2516
2517                 ilk_merge_wm_level(dev, level, wm);
2518
2519                 if (level > last_enabled_level)
2520                         wm->enable = false;
2521                 else if (!ilk_validate_wm_level(level, max, wm))
2522                         /* make sure all following levels get disabled */
2523                         last_enabled_level = level - 1;
2524
2525                 /*
2526                  * The spec says it is preferred to disable
2527                  * FBC WMs instead of disabling a WM level.
2528                  */
2529                 if (wm->fbc_val > max->fbc) {
2530                         if (wm->enable)
2531                                 merged->fbc_wm_enabled = false;
2532                         wm->fbc_val = 0;
2533                 }
2534         }
2535
2536         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
2537         /*
2538          * FIXME this is racy. FBC might get enabled later.
2539          * What we should check here is whether FBC can be
2540          * enabled sometime later.
2541          */
2542         if (IS_GEN5(dev) && !merged->fbc_wm_enabled &&
2543             intel_fbc_enabled(dev_priv)) {
2544                 for (level = 2; level <= max_level; level++) {
2545                         struct intel_wm_level *wm = &merged->wm[level];
2546
2547                         wm->enable = false;
2548                 }
2549         }
2550 }
2551
2552 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
2553 {
2554         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
2555         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
2556 }
2557
2558 /* The value we need to program into the WM_LPx latency field */
2559 static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
2560 {
2561         struct drm_i915_private *dev_priv = dev->dev_private;
2562
2563         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2564                 return 2 * level;
2565         else
2566                 return dev_priv->wm.pri_latency[level];
2567 }
2568
2569 static void ilk_compute_wm_results(struct drm_device *dev,
2570                                    const struct intel_pipe_wm *merged,
2571                                    enum intel_ddb_partitioning partitioning,
2572                                    struct ilk_wm_values *results)
2573 {
2574         struct intel_crtc *intel_crtc;
2575         int level, wm_lp;
2576
2577         results->enable_fbc_wm = merged->fbc_wm_enabled;
2578         results->partitioning = partitioning;
2579
2580         /* LP1+ register values */
2581         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2582                 const struct intel_wm_level *r;
2583
2584                 level = ilk_wm_lp_to_level(wm_lp, merged);
2585
2586                 r = &merged->wm[level];
2587
2588                 /*
2589                  * Maintain the watermark values even if the level is
2590                  * disabled. Doing otherwise could cause underruns.
2591                  */
2592                 results->wm_lp[wm_lp - 1] =
2593                         (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
2594                         (r->pri_val << WM1_LP_SR_SHIFT) |
2595                         r->cur_val;
2596
2597                 if (r->enable)
2598                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
2599
2600                 if (INTEL_INFO(dev)->gen >= 8)
2601                         results->wm_lp[wm_lp - 1] |=
2602                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
2603                 else
2604                         results->wm_lp[wm_lp - 1] |=
2605                                 r->fbc_val << WM1_LP_FBC_SHIFT;
2606
2607                 /*
2608                  * Always set WM1S_LP_EN when spr_val != 0, even if the
2609                  * level is disabled. Doing otherwise could cause underruns.
2610                  */
2611                 if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
2612                         WARN_ON(wm_lp != 1);
2613                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
2614                 } else
2615                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
2616         }
2617
2618         /* LP0 register values */
2619         for_each_intel_crtc(dev, intel_crtc) {
2620                 enum pipe pipe = intel_crtc->pipe;
2621                 const struct intel_wm_level *r =
2622                         &intel_crtc->wm.active.wm[0];
2623
2624                 if (WARN_ON(!r->enable))
2625                         continue;
2626
2627                 results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
2628
2629                 results->wm_pipe[pipe] =
2630                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
2631                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
2632                         r->cur_val;
2633         }
2634 }
2635
2636 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
2637  * case both are at the same level. Prefer r1 in case they're the same. */
2638 static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
2639                                                   struct intel_pipe_wm *r1,
2640                                                   struct intel_pipe_wm *r2)
2641 {
2642         int level, max_level = ilk_wm_max_level(dev);
2643         int level1 = 0, level2 = 0;
2644
2645         for (level = 1; level <= max_level; level++) {
2646                 if (r1->wm[level].enable)
2647                         level1 = level;
2648                 if (r2->wm[level].enable)
2649                         level2 = level;
2650         }
2651
2652         if (level1 == level2) {
2653                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
2654                         return r2;
2655                 else
2656                         return r1;
2657         } else if (level1 > level2) {
2658                 return r1;
2659         } else {
2660                 return r2;
2661         }
2662 }
2663
2664 /* dirty bits used to track which watermarks need changes */
2665 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
2666 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
2667 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
2668 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
2669 #define WM_DIRTY_FBC (1 << 24)
2670 #define WM_DIRTY_DDB (1 << 25)
2671
2672 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
2673                                          const struct ilk_wm_values *old,
2674                                          const struct ilk_wm_values *new)
2675 {
2676         unsigned int dirty = 0;
2677         enum pipe pipe;
2678         int wm_lp;
2679
2680         for_each_pipe(dev_priv, pipe) {
2681                 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
2682                         dirty |= WM_DIRTY_LINETIME(pipe);
2683                         /* Must disable LP1+ watermarks too */
2684                         dirty |= WM_DIRTY_LP_ALL;
2685                 }
2686
2687                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
2688                         dirty |= WM_DIRTY_PIPE(pipe);
2689                         /* Must disable LP1+ watermarks too */
2690                         dirty |= WM_DIRTY_LP_ALL;
2691                 }
2692         }
2693
2694         if (old->enable_fbc_wm != new->enable_fbc_wm) {
2695                 dirty |= WM_DIRTY_FBC;
2696                 /* Must disable LP1+ watermarks too */
2697                 dirty |= WM_DIRTY_LP_ALL;
2698         }
2699
2700         if (old->partitioning != new->partitioning) {
2701                 dirty |= WM_DIRTY_DDB;
2702                 /* Must disable LP1+ watermarks too */
2703                 dirty |= WM_DIRTY_LP_ALL;
2704         }
2705
2706         /* LP1+ watermarks already deemed dirty, no need to continue */
2707         if (dirty & WM_DIRTY_LP_ALL)
2708                 return dirty;
2709
2710         /* Find the lowest numbered LP1+ watermark in need of an update... */
2711         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2712                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
2713                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
2714                         break;
2715         }
2716
2717         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
2718         for (; wm_lp <= 3; wm_lp++)
2719                 dirty |= WM_DIRTY_LP(wm_lp);
2720
2721         return dirty;
2722 }
2723
2724 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
2725                                unsigned int dirty)
2726 {
2727         struct ilk_wm_values *previous = &dev_priv->wm.hw;
2728         bool changed = false;
2729
2730         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
2731                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
2732                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
2733                 changed = true;
2734         }
2735         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
2736                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
2737                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
2738                 changed = true;
2739         }
2740         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
2741                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
2742                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
2743                 changed = true;
2744         }
2745
2746         /*
2747          * Don't touch WM1S_LP_EN here.
2748          * Doing so could cause underruns.
2749          */
2750
2751         return changed;
2752 }
2753
2754 /*
2755  * The spec says we shouldn't write when we don't need, because every write
2756  * causes WMs to be re-evaluated, expending some power.
2757  */
2758 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2759                                 struct ilk_wm_values *results)
2760 {
2761         struct drm_device *dev = dev_priv->dev;
2762         struct ilk_wm_values *previous = &dev_priv->wm.hw;
2763         unsigned int dirty;
2764         uint32_t val;
2765
2766         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
2767         if (!dirty)
2768                 return;
2769
2770         _ilk_disable_lp_wm(dev_priv, dirty);
2771
2772         if (dirty & WM_DIRTY_PIPE(PIPE_A))
2773                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
2774         if (dirty & WM_DIRTY_PIPE(PIPE_B))
2775                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
2776         if (dirty & WM_DIRTY_PIPE(PIPE_C))
2777                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
2778
2779         if (dirty & WM_DIRTY_LINETIME(PIPE_A))
2780                 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
2781         if (dirty & WM_DIRTY_LINETIME(PIPE_B))
2782                 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
2783         if (dirty & WM_DIRTY_LINETIME(PIPE_C))
2784                 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
2785
2786         if (dirty & WM_DIRTY_DDB) {
2787                 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2788                         val = I915_READ(WM_MISC);
2789                         if (results->partitioning == INTEL_DDB_PART_1_2)
2790                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
2791                         else
2792                                 val |= WM_MISC_DATA_PARTITION_5_6;
2793                         I915_WRITE(WM_MISC, val);
2794                 } else {
2795                         val = I915_READ(DISP_ARB_CTL2);
2796                         if (results->partitioning == INTEL_DDB_PART_1_2)
2797                                 val &= ~DISP_DATA_PARTITION_5_6;
2798                         else
2799                                 val |= DISP_DATA_PARTITION_5_6;
2800                         I915_WRITE(DISP_ARB_CTL2, val);
2801                 }
2802         }
2803
2804         if (dirty & WM_DIRTY_FBC) {
2805                 val = I915_READ(DISP_ARB_CTL);
2806                 if (results->enable_fbc_wm)
2807                         val &= ~DISP_FBC_WM_DIS;
2808                 else
2809                         val |= DISP_FBC_WM_DIS;
2810                 I915_WRITE(DISP_ARB_CTL, val);
2811         }
2812
2813         if (dirty & WM_DIRTY_LP(1) &&
2814             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
2815                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
2816
2817         if (INTEL_INFO(dev)->gen >= 7) {
2818                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
2819                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
2820                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
2821                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
2822         }
2823
2824         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
2825                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
2826         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
2827                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
2828         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
2829                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
2830
2831         dev_priv->wm.hw = *results;
2832 }
2833
2834 static bool ilk_disable_lp_wm(struct drm_device *dev)
2835 {
2836         struct drm_i915_private *dev_priv = dev->dev_private;
2837
2838         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
2839 }
2840
2841 /*
2842  * On gen9, we need to allocate Display Data Buffer (DDB) portions to the
2843  * different active planes.
2844  */
2845
2846 #define SKL_DDB_SIZE            896     /* in blocks */
2847 #define BXT_DDB_SIZE            512
2848
2849 static void
2850 skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
2851                                    struct drm_crtc *for_crtc,
2852                                    const struct intel_wm_config *config,
2853                                    const struct skl_pipe_wm_parameters *params,
2854                                    struct skl_ddb_entry *alloc /* out */)
2855 {
2856         struct drm_crtc *crtc;
2857         unsigned int pipe_size, ddb_size;
2858         int nth_active_pipe;
2859
2860         if (!params->active) {
2861                 alloc->start = 0;
2862                 alloc->end = 0;
2863                 return;
2864         }
2865
2866         if (IS_BROXTON(dev))
2867                 ddb_size = BXT_DDB_SIZE;
2868         else
2869                 ddb_size = SKL_DDB_SIZE;
2870
2871         ddb_size -= 4; /* 4 blocks for bypass path allocation */
2872
2873         nth_active_pipe = 0;
2874         for_each_crtc(dev, crtc) {
2875                 if (!to_intel_crtc(crtc)->active)
2876                         continue;
2877
2878                 if (crtc == for_crtc)
2879                         break;
2880
2881                 nth_active_pipe++;
2882         }
2883
2884         pipe_size = ddb_size / config->num_pipes_active;
2885         alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
2886         alloc->end = alloc->start + pipe_size;
2887 }
2888
2889 static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
2890 {
2891         if (config->num_pipes_active == 1)
2892                 return 32;
2893
2894         return 8;
2895 }
2896
2897 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
2898 {
2899         entry->start = reg & 0x3ff;
2900         entry->end = (reg >> 16) & 0x3ff;
2901         if (entry->end)
2902                 entry->end += 1;
2903 }
2904
2905 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
2906                           struct skl_ddb_allocation *ddb /* out */)
2907 {
2908         enum pipe pipe;
2909         int plane;
2910         u32 val;
2911
2912         for_each_pipe(dev_priv, pipe) {
2913                 for_each_plane(dev_priv, pipe, plane) {
2914                         val = I915_READ(PLANE_BUF_CFG(pipe, plane));
2915                         skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
2916                                                    val);
2917                 }
2918
2919                 val = I915_READ(CUR_BUF_CFG(pipe));
2920                 skl_ddb_entry_init_from_hw(&ddb->cursor[pipe], val);
2921         }
2922 }
2923
2924 static unsigned int
2925 skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p, int y)
2926 {
2927
2928         /* for planar format */
2929         if (p->y_bytes_per_pixel) {
2930                 if (y)  /* y-plane data rate */
2931                         return p->horiz_pixels * p->vert_pixels * p->y_bytes_per_pixel;
2932                 else    /* uv-plane data rate */
2933                         return (p->horiz_pixels/2) * (p->vert_pixels/2) * p->bytes_per_pixel;
2934         }
2935
2936         /* for packed formats */
2937         return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel;
2938 }
2939
2940 /*
2941  * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
2942  * a 8192x4096@32bpp framebuffer:
2943  *   3 * 4096 * 8192  * 4 < 2^32
2944  */
2945 static unsigned int
2946 skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc,
2947                                  const struct skl_pipe_wm_parameters *params)
2948 {
2949         unsigned int total_data_rate = 0;
2950         int plane;
2951
2952         for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
2953                 const struct intel_plane_wm_parameters *p;
2954
2955                 p = &params->plane[plane];
2956                 if (!p->enabled)
2957                         continue;
2958
2959                 total_data_rate += skl_plane_relative_data_rate(p, 0); /* packed/uv */
2960                 if (p->y_bytes_per_pixel) {
2961                         total_data_rate += skl_plane_relative_data_rate(p, 1); /* y-plane */
2962                 }
2963         }
2964
2965         return total_data_rate;
2966 }
2967
2968 static void
2969 skl_allocate_pipe_ddb(struct drm_crtc *crtc,
2970                       const struct intel_wm_config *config,
2971                       const struct skl_pipe_wm_parameters *params,
2972                       struct skl_ddb_allocation *ddb /* out */)
2973 {
2974         struct drm_device *dev = crtc->dev;
2975         struct drm_i915_private *dev_priv = dev->dev_private;
2976         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2977         enum pipe pipe = intel_crtc->pipe;
2978         struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
2979         uint16_t alloc_size, start, cursor_blocks;
2980         uint16_t minimum[I915_MAX_PLANES];
2981         uint16_t y_minimum[I915_MAX_PLANES];
2982         unsigned int total_data_rate;
2983         int plane;
2984
2985         skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc);
2986         alloc_size = skl_ddb_entry_size(alloc);
2987         if (alloc_size == 0) {
2988                 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
2989                 memset(&ddb->cursor[pipe], 0, sizeof(ddb->cursor[pipe]));
2990                 return;
2991         }
2992
2993         cursor_blocks = skl_cursor_allocation(config);
2994         ddb->cursor[pipe].start = alloc->end - cursor_blocks;
2995         ddb->cursor[pipe].end = alloc->end;
2996
2997         alloc_size -= cursor_blocks;
2998         alloc->end -= cursor_blocks;
2999
3000         /* 1. Allocate the mininum required blocks for each active plane */
3001         for_each_plane(dev_priv, pipe, plane) {
3002                 const struct intel_plane_wm_parameters *p;
3003
3004                 p = &params->plane[plane];
3005                 if (!p->enabled)
3006                         continue;
3007
3008                 minimum[plane] = 8;
3009                 alloc_size -= minimum[plane];
3010                 y_minimum[plane] = p->y_bytes_per_pixel ? 8 : 0;
3011                 alloc_size -= y_minimum[plane];
3012         }
3013
3014         /*
3015          * 2. Distribute the remaining space in proportion to the amount of
3016          * data each plane needs to fetch from memory.
3017          *
3018          * FIXME: we may not allocate every single block here.
3019          */
3020         total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params);
3021
3022         start = alloc->start;
3023         for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
3024                 const struct intel_plane_wm_parameters *p;
3025                 unsigned int data_rate, y_data_rate;
3026                 uint16_t plane_blocks, y_plane_blocks = 0;
3027
3028                 p = &params->plane[plane];
3029                 if (!p->enabled)
3030                         continue;
3031
3032                 data_rate = skl_plane_relative_data_rate(p, 0);
3033
3034                 /*
3035                  * allocation for (packed formats) or (uv-plane part of planar format):
3036                  * promote the expression to 64 bits to avoid overflowing, the
3037                  * result is < available as data_rate / total_data_rate < 1
3038                  */
3039                 plane_blocks = minimum[plane];
3040                 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
3041                                         total_data_rate);
3042
3043                 ddb->plane[pipe][plane].start = start;
3044                 ddb->plane[pipe][plane].end = start + plane_blocks;
3045
3046                 start += plane_blocks;
3047
3048                 /*
3049                  * allocation for y_plane part of planar format:
3050                  */
3051                 if (p->y_bytes_per_pixel) {
3052                         y_data_rate = skl_plane_relative_data_rate(p, 1);
3053                         y_plane_blocks = y_minimum[plane];
3054                         y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
3055                                                 total_data_rate);
3056
3057                         ddb->y_plane[pipe][plane].start = start;
3058                         ddb->y_plane[pipe][plane].end = start + y_plane_blocks;
3059
3060                         start += y_plane_blocks;
3061                 }
3062
3063         }
3064
3065 }
3066
3067 static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
3068 {
3069         /* TODO: Take into account the scalers once we support them */
3070         return config->base.adjusted_mode.crtc_clock;
3071 }
3072
3073 /*
3074  * The max latency should be 257 (max the punit can code is 255 and we add 2us
3075  * for the read latency) and bytes_per_pixel should always be <= 8, so that
3076  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
3077  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
3078 */
3079 static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
3080                                uint32_t latency)
3081 {
3082         uint32_t wm_intermediate_val, ret;
3083
3084         if (latency == 0)
3085                 return UINT_MAX;
3086
3087         wm_intermediate_val = latency * pixel_rate * bytes_per_pixel / 512;
3088         ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
3089
3090         return ret;
3091 }
3092
3093 static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
3094                                uint32_t horiz_pixels, uint8_t bytes_per_pixel,
3095                                uint64_t tiling, uint32_t latency)
3096 {
3097         uint32_t ret;
3098         uint32_t plane_bytes_per_line, plane_blocks_per_line;
3099         uint32_t wm_intermediate_val;
3100
3101         if (latency == 0)
3102                 return UINT_MAX;
3103
3104         plane_bytes_per_line = horiz_pixels * bytes_per_pixel;
3105
3106         if (tiling == I915_FORMAT_MOD_Y_TILED ||
3107             tiling == I915_FORMAT_MOD_Yf_TILED) {
3108                 plane_bytes_per_line *= 4;
3109                 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
3110                 plane_blocks_per_line /= 4;
3111         } else {
3112                 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
3113         }
3114
3115         wm_intermediate_val = latency * pixel_rate;
3116         ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
3117                                 plane_blocks_per_line;
3118
3119         return ret;
3120 }
3121
3122 static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
3123                                        const struct intel_crtc *intel_crtc)
3124 {
3125         struct drm_device *dev = intel_crtc->base.dev;
3126         struct drm_i915_private *dev_priv = dev->dev_private;
3127         const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
3128         enum pipe pipe = intel_crtc->pipe;
3129
3130         if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe],
3131                    sizeof(new_ddb->plane[pipe])))
3132                 return true;
3133
3134         if (memcmp(&new_ddb->cursor[pipe], &cur_ddb->cursor[pipe],
3135                     sizeof(new_ddb->cursor[pipe])))
3136                 return true;
3137
3138         return false;
3139 }
3140
3141 static void skl_compute_wm_global_parameters(struct drm_device *dev,
3142                                              struct intel_wm_config *config)
3143 {
3144         struct drm_crtc *crtc;
3145         struct drm_plane *plane;
3146
3147         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
3148                 config->num_pipes_active += to_intel_crtc(crtc)->active;
3149
3150         /* FIXME: I don't think we need those two global parameters on SKL */
3151         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
3152                 struct intel_plane *intel_plane = to_intel_plane(plane);
3153
3154                 config->sprites_enabled |= intel_plane->wm.enabled;
3155                 config->sprites_scaled |= intel_plane->wm.scaled;
3156         }
3157 }
3158
3159 static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc,
3160                                            struct skl_pipe_wm_parameters *p)
3161 {
3162         struct drm_device *dev = crtc->dev;
3163         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3164         enum pipe pipe = intel_crtc->pipe;
3165         struct drm_plane *plane;
3166         struct drm_framebuffer *fb;
3167         int i = 1; /* Index for sprite planes start */
3168
3169         p->active = intel_crtc->active;
3170         if (p->active) {
3171                 p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
3172                 p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config);
3173
3174                 fb = crtc->primary->state->fb;
3175                 /* For planar: Bpp is for uv plane, y_Bpp is for y plane */
3176                 if (fb) {
3177                         p->plane[0].enabled = true;
3178                         p->plane[0].bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
3179                                 drm_format_plane_cpp(fb->pixel_format, 1) :
3180                                 drm_format_plane_cpp(fb->pixel_format, 0);
3181                         p->plane[0].y_bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
3182                                 drm_format_plane_cpp(fb->pixel_format, 0) : 0;
3183                         p->plane[0].tiling = fb->modifier[0];
3184                 } else {
3185                         p->plane[0].enabled = false;
3186                         p->plane[0].bytes_per_pixel = 0;
3187                         p->plane[0].y_bytes_per_pixel = 0;
3188                         p->plane[0].tiling = DRM_FORMAT_MOD_NONE;
3189                 }
3190                 p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w;
3191                 p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h;
3192                 p->plane[0].rotation = crtc->primary->state->rotation;
3193
3194                 fb = crtc->cursor->state->fb;
3195                 p->cursor.y_bytes_per_pixel = 0;
3196                 if (fb) {
3197                         p->cursor.enabled = true;
3198                         p->cursor.bytes_per_pixel = fb->bits_per_pixel / 8;
3199                         p->cursor.horiz_pixels = crtc->cursor->state->crtc_w;
3200                         p->cursor.vert_pixels = crtc->cursor->state->crtc_h;
3201                 } else {
3202                         p->cursor.enabled = false;
3203                         p->cursor.bytes_per_pixel = 0;
3204                         p->cursor.horiz_pixels = 64;
3205                         p->cursor.vert_pixels = 64;
3206                 }
3207         }
3208
3209         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
3210                 struct intel_plane *intel_plane = to_intel_plane(plane);
3211
3212                 if (intel_plane->pipe == pipe &&
3213                         plane->type == DRM_PLANE_TYPE_OVERLAY)
3214                         p->plane[i++] = intel_plane->wm;
3215         }
3216 }
3217
3218 static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
3219                                  struct skl_pipe_wm_parameters *p,
3220                                  struct intel_plane_wm_parameters *p_params,
3221                                  uint16_t ddb_allocation,
3222                                  int level,
3223                                  uint16_t *out_blocks, /* out */
3224                                  uint8_t *out_lines /* out */)
3225 {
3226         uint32_t latency = dev_priv->wm.skl_latency[level];
3227         uint32_t method1, method2;
3228         uint32_t plane_bytes_per_line, plane_blocks_per_line;
3229         uint32_t res_blocks, res_lines;
3230         uint32_t selected_result;
3231         uint8_t bytes_per_pixel;
3232
3233         if (latency == 0 || !p->active || !p_params->enabled)
3234                 return false;
3235
3236         bytes_per_pixel = p_params->y_bytes_per_pixel ?
3237                 p_params->y_bytes_per_pixel :
3238                 p_params->bytes_per_pixel;
3239         method1 = skl_wm_method1(p->pixel_rate,
3240                                  bytes_per_pixel,
3241                                  latency);
3242         method2 = skl_wm_method2(p->pixel_rate,
3243                                  p->pipe_htotal,
3244                                  p_params->horiz_pixels,
3245                                  bytes_per_pixel,
3246                                  p_params->tiling,
3247                                  latency);
3248
3249         plane_bytes_per_line = p_params->horiz_pixels * bytes_per_pixel;
3250         plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
3251
3252         if (p_params->tiling == I915_FORMAT_MOD_Y_TILED ||
3253             p_params->tiling == I915_FORMAT_MOD_Yf_TILED) {
3254                 uint32_t min_scanlines = 4;
3255                 uint32_t y_tile_minimum;
3256                 if (intel_rotation_90_or_270(p_params->rotation)) {
3257                         switch (p_params->bytes_per_pixel) {
3258                         case 1:
3259                                 min_scanlines = 16;
3260                                 break;
3261                         case 2:
3262                                 min_scanlines = 8;
3263                                 break;
3264                         case 8:
3265                                 WARN(1, "Unsupported pixel depth for rotation");
3266                         }
3267                 }
3268                 y_tile_minimum = plane_blocks_per_line * min_scanlines;
3269                 selected_result = max(method2, y_tile_minimum);
3270         } else {
3271                 if ((ddb_allocation / plane_blocks_per_line) >= 1)
3272                         selected_result = min(method1, method2);
3273                 else
3274                         selected_result = method1;
3275         }
3276
3277         res_blocks = selected_result + 1;
3278         res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line);
3279
3280         if (level >= 1 && level <= 7) {
3281                 if (p_params->tiling == I915_FORMAT_MOD_Y_TILED ||
3282                     p_params->tiling == I915_FORMAT_MOD_Yf_TILED)
3283                         res_lines += 4;
3284                 else
3285                         res_blocks++;
3286         }
3287
3288         if (res_blocks >= ddb_allocation || res_lines > 31)
3289                 return false;
3290
3291         *out_blocks = res_blocks;
3292         *out_lines = res_lines;
3293
3294         return true;
3295 }
3296
3297 static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
3298                                  struct skl_ddb_allocation *ddb,
3299                                  struct skl_pipe_wm_parameters *p,
3300                                  enum pipe pipe,
3301                                  int level,
3302                                  int num_planes,
3303                                  struct skl_wm_level *result)
3304 {
3305         uint16_t ddb_blocks;
3306         int i;
3307
3308         for (i = 0; i < num_planes; i++) {
3309                 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
3310
3311                 result->plane_en[i] = skl_compute_plane_wm(dev_priv,
3312                                                 p, &p->plane[i],
3313                                                 ddb_blocks,
3314                                                 level,
3315                                                 &result->plane_res_b[i],
3316                                                 &result->plane_res_l[i]);
3317         }
3318
3319         ddb_blocks = skl_ddb_entry_size(&ddb->cursor[pipe]);
3320         result->cursor_en = skl_compute_plane_wm(dev_priv, p, &p->cursor,
3321                                                  ddb_blocks, level,
3322                                                  &result->cursor_res_b,
3323                                                  &result->cursor_res_l);
3324 }
3325
3326 static uint32_t
3327 skl_compute_linetime_wm(struct drm_crtc *crtc, struct skl_pipe_wm_parameters *p)
3328 {
3329         if (!to_intel_crtc(crtc)->active)
3330                 return 0;
3331
3332         if (WARN_ON(p->pixel_rate == 0))
3333                 return 0;
3334
3335         return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate);
3336 }
3337
3338 static void skl_compute_transition_wm(struct drm_crtc *crtc,
3339                                       struct skl_pipe_wm_parameters *params,
3340                                       struct skl_wm_level *trans_wm /* out */)
3341 {
3342         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3343         int i;
3344
3345         if (!params->active)
3346                 return;
3347
3348         /* Until we know more, just disable transition WMs */
3349         for (i = 0; i < intel_num_planes(intel_crtc); i++)
3350                 trans_wm->plane_en[i] = false;
3351         trans_wm->cursor_en = false;
3352 }
3353
3354 static void skl_compute_pipe_wm(struct drm_crtc *crtc,
3355                                 struct skl_ddb_allocation *ddb,
3356                                 struct skl_pipe_wm_parameters *params,
3357                                 struct skl_pipe_wm *pipe_wm)
3358 {
3359         struct drm_device *dev = crtc->dev;
3360         const struct drm_i915_private *dev_priv = dev->dev_private;
3361         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3362         int level, max_level = ilk_wm_max_level(dev);
3363
3364         for (level = 0; level <= max_level; level++) {
3365                 skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe,
3366                                      level, intel_num_planes(intel_crtc),
3367                                      &pipe_wm->wm[level]);
3368         }
3369         pipe_wm->linetime = skl_compute_linetime_wm(crtc, params);
3370
3371         skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm);
3372 }
3373
3374 static void skl_compute_wm_results(struct drm_device *dev,
3375                                    struct skl_pipe_wm_parameters *p,
3376                                    struct skl_pipe_wm *p_wm,
3377                                    struct skl_wm_values *r,
3378                                    struct intel_crtc *intel_crtc)
3379 {
3380         int level, max_level = ilk_wm_max_level(dev);
3381         enum pipe pipe = intel_crtc->pipe;
3382         uint32_t temp;
3383         int i;
3384
3385         for (level = 0; level <= max_level; level++) {
3386                 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3387                         temp = 0;
3388
3389                         temp |= p_wm->wm[level].plane_res_l[i] <<
3390                                         PLANE_WM_LINES_SHIFT;
3391                         temp |= p_wm->wm[level].plane_res_b[i];
3392                         if (p_wm->wm[level].plane_en[i])
3393                                 temp |= PLANE_WM_EN;
3394
3395                         r->plane[pipe][i][level] = temp;
3396                 }
3397
3398                 temp = 0;
3399
3400                 temp |= p_wm->wm[level].cursor_res_l << PLANE_WM_LINES_SHIFT;
3401                 temp |= p_wm->wm[level].cursor_res_b;
3402
3403                 if (p_wm->wm[level].cursor_en)
3404                         temp |= PLANE_WM_EN;
3405
3406                 r->cursor[pipe][level] = temp;
3407
3408         }
3409
3410         /* transition WMs */
3411         for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3412                 temp = 0;
3413                 temp |= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
3414                 temp |= p_wm->trans_wm.plane_res_b[i];
3415                 if (p_wm->trans_wm.plane_en[i])
3416                         temp |= PLANE_WM_EN;
3417
3418                 r->plane_trans[pipe][i] = temp;
3419         }
3420
3421         temp = 0;
3422         temp |= p_wm->trans_wm.cursor_res_l << PLANE_WM_LINES_SHIFT;
3423         temp |= p_wm->trans_wm.cursor_res_b;
3424         if (p_wm->trans_wm.cursor_en)
3425                 temp |= PLANE_WM_EN;
3426
3427         r->cursor_trans[pipe] = temp;
3428
3429         r->wm_linetime[pipe] = p_wm->linetime;
3430 }
3431
3432 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg,
3433                                 const struct skl_ddb_entry *entry)
3434 {
3435         if (entry->end)
3436                 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
3437         else
3438                 I915_WRITE(reg, 0);
3439 }
3440
3441 static void skl_write_wm_values(struct drm_i915_private *dev_priv,
3442                                 const struct skl_wm_values *new)
3443 {
3444         struct drm_device *dev = dev_priv->dev;
3445         struct intel_crtc *crtc;
3446
3447         list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
3448                 int i, level, max_level = ilk_wm_max_level(dev);
3449                 enum pipe pipe = crtc->pipe;
3450
3451                 if (!new->dirty[pipe])
3452                         continue;
3453
3454                 I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
3455
3456                 for (level = 0; level <= max_level; level++) {
3457                         for (i = 0; i < intel_num_planes(crtc); i++)
3458                                 I915_WRITE(PLANE_WM(pipe, i, level),
3459                                            new->plane[pipe][i][level]);
3460                         I915_WRITE(CUR_WM(pipe, level),
3461                                    new->cursor[pipe][level]);
3462                 }
3463                 for (i = 0; i < intel_num_planes(crtc); i++)
3464                         I915_WRITE(PLANE_WM_TRANS(pipe, i),
3465                                    new->plane_trans[pipe][i]);
3466                 I915_WRITE(CUR_WM_TRANS(pipe), new->cursor_trans[pipe]);
3467
3468                 for (i = 0; i < intel_num_planes(crtc); i++) {
3469                         skl_ddb_entry_write(dev_priv,
3470                                             PLANE_BUF_CFG(pipe, i),
3471                                             &new->ddb.plane[pipe][i]);
3472                         skl_ddb_entry_write(dev_priv,
3473                                             PLANE_NV12_BUF_CFG(pipe, i),
3474                                             &new->ddb.y_plane[pipe][i]);
3475                 }
3476
3477                 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
3478                                     &new->ddb.cursor[pipe]);
3479         }
3480 }
3481
3482 /*
3483  * When setting up a new DDB allocation arrangement, we need to correctly
3484  * sequence the times at which the new allocations for the pipes are taken into
3485  * account or we'll have pipes fetching from space previously allocated to
3486  * another pipe.
3487  *
3488  * Roughly the sequence looks like:
3489  *  1. re-allocate the pipe(s) with the allocation being reduced and not
3490  *     overlapping with a previous light-up pipe (another way to put it is:
3491  *     pipes with their new allocation strickly included into their old ones).
3492  *  2. re-allocate the other pipes that get their allocation reduced
3493  *  3. allocate the pipes having their allocation increased
3494  *
3495  * Steps 1. and 2. are here to take care of the following case:
3496  * - Initially DDB looks like this:
3497  *     |   B    |   C    |
3498  * - enable pipe A.
3499  * - pipe B has a reduced DDB allocation that overlaps with the old pipe C
3500  *   allocation
3501  *     |  A  |  B  |  C  |
3502  *
3503  * We need to sequence the re-allocation: C, B, A (and not B, C, A).
3504  */
3505
3506 static void
3507 skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
3508 {
3509         int plane;
3510
3511         DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
3512
3513         for_each_plane(dev_priv, pipe, plane) {
3514                 I915_WRITE(PLANE_SURF(pipe, plane),
3515                            I915_READ(PLANE_SURF(pipe, plane)));
3516         }
3517         I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
3518 }
3519
3520 static bool
3521 skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
3522                             const struct skl_ddb_allocation *new,
3523                             enum pipe pipe)
3524 {
3525         uint16_t old_size, new_size;
3526
3527         old_size = skl_ddb_entry_size(&old->pipe[pipe]);
3528         new_size = skl_ddb_entry_size(&new->pipe[pipe]);
3529
3530         return old_size != new_size &&
3531                new->pipe[pipe].start >= old->pipe[pipe].start &&
3532                new->pipe[pipe].end <= old->pipe[pipe].end;
3533 }
3534
3535 static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
3536                                 struct skl_wm_values *new_values)
3537 {
3538         struct drm_device *dev = dev_priv->dev;
3539         struct skl_ddb_allocation *cur_ddb, *new_ddb;
3540         bool reallocated[I915_MAX_PIPES] = {};
3541         struct intel_crtc *crtc;
3542         enum pipe pipe;
3543
3544         new_ddb = &new_values->ddb;
3545         cur_ddb = &dev_priv->wm.skl_hw.ddb;
3546
3547         /*
3548          * First pass: flush the pipes with the new allocation contained into
3549          * the old space.
3550          *
3551          * We'll wait for the vblank on those pipes to ensure we can safely
3552          * re-allocate the freed space without this pipe fetching from it.
3553          */
3554         for_each_intel_crtc(dev, crtc) {
3555                 if (!crtc->active)
3556                         continue;
3557
3558                 pipe = crtc->pipe;
3559
3560                 if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
3561                         continue;
3562
3563                 skl_wm_flush_pipe(dev_priv, pipe, 1);
3564                 intel_wait_for_vblank(dev, pipe);
3565
3566                 reallocated[pipe] = true;
3567         }
3568
3569
3570         /*
3571          * Second pass: flush the pipes that are having their allocation
3572          * reduced, but overlapping with a previous allocation.
3573          *
3574          * Here as well we need to wait for the vblank to make sure the freed
3575          * space is not used anymore.
3576          */
3577         for_each_intel_crtc(dev, crtc) {
3578                 if (!crtc->active)
3579                         continue;
3580
3581                 pipe = crtc->pipe;
3582
3583                 if (reallocated[pipe])
3584                         continue;
3585
3586                 if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
3587                     skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
3588                         skl_wm_flush_pipe(dev_priv, pipe, 2);
3589                         intel_wait_for_vblank(dev, pipe);
3590                         reallocated[pipe] = true;
3591                 }
3592         }
3593
3594         /*
3595          * Third pass: flush the pipes that got more space allocated.
3596          *
3597          * We don't need to actively wait for the update here, next vblank
3598          * will just get more DDB space with the correct WM values.
3599          */
3600         for_each_intel_crtc(dev, crtc) {
3601                 if (!crtc->active)
3602                         continue;
3603
3604                 pipe = crtc->pipe;
3605
3606                 /*
3607                  * At this point, only the pipes more space than before are
3608                  * left to re-allocate.
3609                  */
3610                 if (reallocated[pipe])
3611                         continue;
3612
3613                 skl_wm_flush_pipe(dev_priv, pipe, 3);
3614         }
3615 }
3616
3617 static bool skl_update_pipe_wm(struct drm_crtc *crtc,
3618                                struct skl_pipe_wm_parameters *params,
3619                                struct intel_wm_config *config,
3620                                struct skl_ddb_allocation *ddb, /* out */
3621                                struct skl_pipe_wm *pipe_wm /* out */)
3622 {
3623         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3624
3625         skl_compute_wm_pipe_parameters(crtc, params);
3626         skl_allocate_pipe_ddb(crtc, config, params, ddb);
3627         skl_compute_pipe_wm(crtc, ddb, params, pipe_wm);
3628
3629         if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm)))
3630                 return false;
3631
3632         intel_crtc->wm.skl_active = *pipe_wm;
3633
3634         return true;
3635 }
3636
3637 static void skl_update_other_pipe_wm(struct drm_device *dev,
3638                                      struct drm_crtc *crtc,
3639                                      struct intel_wm_config *config,
3640                                      struct skl_wm_values *r)
3641 {
3642         struct intel_crtc *intel_crtc;
3643         struct intel_crtc *this_crtc = to_intel_crtc(crtc);
3644
3645         /*
3646          * If the WM update hasn't changed the allocation for this_crtc (the
3647          * crtc we are currently computing the new WM values for), other
3648          * enabled crtcs will keep the same allocation and we don't need to
3649          * recompute anything for them.
3650          */
3651         if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
3652                 return;
3653
3654         /*
3655          * Otherwise, because of this_crtc being freshly enabled/disabled, the
3656          * other active pipes need new DDB allocation and WM values.
3657          */
3658         list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
3659                                 base.head) {
3660                 struct skl_pipe_wm_parameters params = {};
3661                 struct skl_pipe_wm pipe_wm = {};
3662                 bool wm_changed;
3663
3664                 if (this_crtc->pipe == intel_crtc->pipe)
3665                         continue;
3666
3667                 if (!intel_crtc->active)
3668                         continue;
3669
3670                 wm_changed = skl_update_pipe_wm(&intel_crtc->base,
3671                                                 &params, config,
3672                                                 &r->ddb, &pipe_wm);
3673
3674                 /*
3675                  * If we end up re-computing the other pipe WM values, it's
3676                  * because it was really needed, so we expect the WM values to
3677                  * be different.
3678                  */
3679                 WARN_ON(!wm_changed);
3680
3681                 skl_compute_wm_results(dev, &params, &pipe_wm, r, intel_crtc);
3682                 r->dirty[intel_crtc->pipe] = true;
3683         }
3684 }
3685
3686 static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe)
3687 {
3688         watermarks->wm_linetime[pipe] = 0;
3689         memset(watermarks->plane[pipe], 0,
3690                sizeof(uint32_t) * 8 * I915_MAX_PLANES);
3691         memset(watermarks->cursor[pipe], 0, sizeof(uint32_t) * 8);
3692         memset(watermarks->plane_trans[pipe],
3693                0, sizeof(uint32_t) * I915_MAX_PLANES);
3694         watermarks->cursor_trans[pipe] = 0;
3695
3696         /* Clear ddb entries for pipe */
3697         memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry));
3698         memset(&watermarks->ddb.plane[pipe], 0,
3699                sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
3700         memset(&watermarks->ddb.y_plane[pipe], 0,
3701                sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
3702         memset(&watermarks->ddb.cursor[pipe], 0, sizeof(struct skl_ddb_entry));
3703
3704 }
3705
3706 static void skl_update_wm(struct drm_crtc *crtc)
3707 {
3708         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3709         struct drm_device *dev = crtc->dev;
3710         struct drm_i915_private *dev_priv = dev->dev_private;
3711         struct skl_pipe_wm_parameters params = {};
3712         struct skl_wm_values *results = &dev_priv->wm.skl_results;
3713         struct skl_pipe_wm pipe_wm = {};
3714         struct intel_wm_config config = {};
3715
3716
3717         /* Clear all dirty flags */
3718         memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES);
3719
3720         skl_clear_wm(results, intel_crtc->pipe);
3721
3722         skl_compute_wm_global_parameters(dev, &config);
3723
3724         if (!skl_update_pipe_wm(crtc, &params, &config,
3725                                 &results->ddb, &pipe_wm))
3726                 return;
3727
3728         skl_compute_wm_results(dev, &params, &pipe_wm, results, intel_crtc);
3729         results->dirty[intel_crtc->pipe] = true;
3730
3731         skl_update_other_pipe_wm(dev, crtc, &config, results);
3732         skl_write_wm_values(dev_priv, results);
3733         skl_flush_wm_values(dev_priv, results);
3734
3735         /* store the new configuration */
3736         dev_priv->wm.skl_hw = *results;
3737 }
3738
3739 static void
3740 skl_update_sprite_wm(struct drm_plane *plane, struct drm_crtc *crtc,
3741                      uint32_t sprite_width, uint32_t sprite_height,
3742                      int pixel_size, bool enabled, bool scaled)
3743 {
3744         struct intel_plane *intel_plane = to_intel_plane(plane);
3745         struct drm_framebuffer *fb = plane->state->fb;
3746
3747         intel_plane->wm.enabled = enabled;
3748         intel_plane->wm.scaled = scaled;
3749         intel_plane->wm.horiz_pixels = sprite_width;
3750         intel_plane->wm.vert_pixels = sprite_height;
3751         intel_plane->wm.tiling = DRM_FORMAT_MOD_NONE;
3752
3753         /* For planar: Bpp is for UV plane, y_Bpp is for Y plane */
3754         intel_plane->wm.bytes_per_pixel =
3755                 (fb && fb->pixel_format == DRM_FORMAT_NV12) ?
3756                 drm_format_plane_cpp(plane->state->fb->pixel_format, 1) : pixel_size;
3757         intel_plane->wm.y_bytes_per_pixel =
3758                 (fb && fb->pixel_format == DRM_FORMAT_NV12) ?
3759                 drm_format_plane_cpp(plane->state->fb->pixel_format, 0) : 0;
3760
3761         /*
3762          * Framebuffer can be NULL on plane disable, but it does not
3763          * matter for watermarks if we assume no tiling in that case.
3764          */
3765         if (fb)
3766                 intel_plane->wm.tiling = fb->modifier[0];
3767         intel_plane->wm.rotation = plane->state->rotation;
3768
3769         skl_update_wm(crtc);
3770 }
3771
3772 static void ilk_update_wm(struct drm_crtc *crtc)
3773 {
3774         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3775         struct drm_device *dev = crtc->dev;
3776         struct drm_i915_private *dev_priv = dev->dev_private;
3777         struct ilk_wm_maximums max;
3778         struct ilk_pipe_wm_parameters params = {};
3779         struct ilk_wm_values results = {};
3780         enum intel_ddb_partitioning partitioning;
3781         struct intel_pipe_wm pipe_wm = {};
3782         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3783         struct intel_wm_config config = {};
3784
3785         ilk_compute_wm_parameters(crtc, &params);
3786
3787         intel_compute_pipe_wm(crtc, &params, &pipe_wm);
3788
3789         if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
3790                 return;
3791
3792         intel_crtc->wm.active = pipe_wm;
3793
3794         ilk_compute_wm_config(dev, &config);
3795
3796         ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
3797         ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
3798
3799         /* 5/6 split only in single pipe config on IVB+ */
3800         if (INTEL_INFO(dev)->gen >= 7 &&
3801             config.num_pipes_active == 1 && config.sprites_enabled) {
3802                 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
3803                 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
3804
3805                 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
3806         } else {
3807                 best_lp_wm = &lp_wm_1_2;
3808         }
3809
3810         partitioning = (best_lp_wm == &lp_wm_1_2) ?
3811                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3812
3813         ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
3814
3815         ilk_write_wm_values(dev_priv, &results);
3816 }
3817
3818 static void
3819 ilk_update_sprite_wm(struct drm_plane *plane,
3820                      struct drm_crtc *crtc,
3821                      uint32_t sprite_width, uint32_t sprite_height,
3822                      int pixel_size, bool enabled, bool scaled)
3823 {
3824         struct drm_device *dev = plane->dev;
3825         struct intel_plane *intel_plane = to_intel_plane(plane);
3826
3827         intel_plane->wm.enabled = enabled;
3828         intel_plane->wm.scaled = scaled;
3829         intel_plane->wm.horiz_pixels = sprite_width;
3830         intel_plane->wm.vert_pixels = sprite_width;
3831         intel_plane->wm.bytes_per_pixel = pixel_size;
3832
3833         /*
3834          * IVB workaround: must disable low power watermarks for at least
3835          * one frame before enabling scaling.  LP watermarks can be re-enabled
3836          * when scaling is disabled.
3837          *
3838          * WaCxSRDisabledForSpriteScaling:ivb
3839          */
3840         if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
3841                 intel_wait_for_vblank(dev, intel_plane->pipe);
3842
3843         ilk_update_wm(crtc);
3844 }
3845
3846 static void skl_pipe_wm_active_state(uint32_t val,
3847                                      struct skl_pipe_wm *active,
3848                                      bool is_transwm,
3849                                      bool is_cursor,
3850                                      int i,
3851                                      int level)
3852 {
3853         bool is_enabled = (val & PLANE_WM_EN) != 0;
3854
3855         if (!is_transwm) {
3856                 if (!is_cursor) {
3857                         active->wm[level].plane_en[i] = is_enabled;
3858                         active->wm[level].plane_res_b[i] =
3859                                         val & PLANE_WM_BLOCKS_MASK;
3860                         active->wm[level].plane_res_l[i] =
3861                                         (val >> PLANE_WM_LINES_SHIFT) &
3862                                                 PLANE_WM_LINES_MASK;
3863                 } else {
3864                         active->wm[level].cursor_en = is_enabled;
3865                         active->wm[level].cursor_res_b =
3866                                         val & PLANE_WM_BLOCKS_MASK;
3867                         active->wm[level].cursor_res_l =
3868                                         (val >> PLANE_WM_LINES_SHIFT) &
3869                                                 PLANE_WM_LINES_MASK;
3870                 }
3871         } else {
3872                 if (!is_cursor) {
3873                         active->trans_wm.plane_en[i] = is_enabled;
3874                         active->trans_wm.plane_res_b[i] =
3875                                         val & PLANE_WM_BLOCKS_MASK;
3876                         active->trans_wm.plane_res_l[i] =
3877                                         (val >> PLANE_WM_LINES_SHIFT) &
3878                                                 PLANE_WM_LINES_MASK;
3879                 } else {
3880                         active->trans_wm.cursor_en = is_enabled;
3881                         active->trans_wm.cursor_res_b =
3882                                         val & PLANE_WM_BLOCKS_MASK;
3883                         active->trans_wm.cursor_res_l =
3884                                         (val >> PLANE_WM_LINES_SHIFT) &
3885                                                 PLANE_WM_LINES_MASK;
3886                 }
3887         }
3888 }
3889
3890 static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3891 {
3892         struct drm_device *dev = crtc->dev;
3893         struct drm_i915_private *dev_priv = dev->dev_private;
3894         struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
3895         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3896         struct skl_pipe_wm *active = &intel_crtc->wm.skl_active;
3897         enum pipe pipe = intel_crtc->pipe;
3898         int level, i, max_level;
3899         uint32_t temp;
3900
3901         max_level = ilk_wm_max_level(dev);
3902
3903         hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3904
3905         for (level = 0; level <= max_level; level++) {
3906                 for (i = 0; i < intel_num_planes(intel_crtc); i++)
3907                         hw->plane[pipe][i][level] =
3908                                         I915_READ(PLANE_WM(pipe, i, level));
3909                 hw->cursor[pipe][level] = I915_READ(CUR_WM(pipe, level));
3910         }
3911
3912         for (i = 0; i < intel_num_planes(intel_crtc); i++)
3913                 hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
3914         hw->cursor_trans[pipe] = I915_READ(CUR_WM_TRANS(pipe));
3915
3916         if (!intel_crtc->active)
3917                 return;
3918
3919         hw->dirty[pipe] = true;
3920
3921         active->linetime = hw->wm_linetime[pipe];
3922
3923         for (level = 0; level <= max_level; level++) {
3924                 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3925                         temp = hw->plane[pipe][i][level];
3926                         skl_pipe_wm_active_state(temp, active, false,
3927                                                 false, i, level);
3928                 }
3929                 temp = hw->cursor[pipe][level];
3930                 skl_pipe_wm_active_state(temp, active, false, true, i, level);
3931         }
3932
3933         for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3934                 temp = hw->plane_trans[pipe][i];
3935                 skl_pipe_wm_active_state(temp, active, true, false, i, 0);
3936         }
3937
3938         temp = hw->cursor_trans[pipe];
3939         skl_pipe_wm_active_state(temp, active, true, true, i, 0);
3940 }
3941
3942 void skl_wm_get_hw_state(struct drm_device *dev)
3943 {
3944         struct drm_i915_private *dev_priv = dev->dev_private;
3945         struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
3946         struct drm_crtc *crtc;
3947
3948         skl_ddb_get_hw_state(dev_priv, ddb);
3949         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
3950                 skl_pipe_wm_get_hw_state(crtc);
3951 }
3952
3953 static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3954 {
3955         struct drm_device *dev = crtc->dev;
3956         struct drm_i915_private *dev_priv = dev->dev_private;
3957         struct ilk_wm_values *hw = &dev_priv->wm.hw;
3958         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3959         struct intel_pipe_wm *active = &intel_crtc->wm.active;
3960         enum pipe pipe = intel_crtc->pipe;
3961         static const unsigned int wm0_pipe_reg[] = {
3962                 [PIPE_A] = WM0_PIPEA_ILK,
3963                 [PIPE_B] = WM0_PIPEB_ILK,
3964                 [PIPE_C] = WM0_PIPEC_IVB,
3965         };
3966
3967         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
3968         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
3969                 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3970
3971         active->pipe_enabled = intel_crtc->active;
3972
3973         if (active->pipe_enabled) {
3974                 u32 tmp = hw->wm_pipe[pipe];
3975
3976                 /*
3977                  * For active pipes LP0 watermark is marked as
3978                  * enabled, and LP1+ watermaks as disabled since
3979                  * we can't really reverse compute them in case
3980                  * multiple pipes are active.
3981                  */
3982                 active->wm[0].enable = true;
3983                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
3984                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
3985                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
3986                 active->linetime = hw->wm_linetime[pipe];
3987         } else {
3988                 int level, max_level = ilk_wm_max_level(dev);
3989
3990                 /*
3991                  * For inactive pipes, all watermark levels
3992                  * should be marked as enabled but zeroed,
3993                  * which is what we'd compute them to.
3994                  */
3995                 for (level = 0; level <= max_level; level++)
3996                         active->wm[level].enable = true;
3997         }
3998 }
3999
4000 #define _FW_WM(value, plane) \
4001         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
4002 #define _FW_WM_VLV(value, plane) \
4003         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
4004
4005 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
4006                                struct vlv_wm_values *wm)
4007 {
4008         enum pipe pipe;
4009         uint32_t tmp;
4010
4011         for_each_pipe(dev_priv, pipe) {
4012                 tmp = I915_READ(VLV_DDL(pipe));
4013
4014                 wm->ddl[pipe].primary =
4015                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
4016                 wm->ddl[pipe].cursor =
4017                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
4018                 wm->ddl[pipe].sprite[0] =
4019                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
4020                 wm->ddl[pipe].sprite[1] =
4021                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
4022         }
4023
4024         tmp = I915_READ(DSPFW1);
4025         wm->sr.plane = _FW_WM(tmp, SR);
4026         wm->pipe[PIPE_B].cursor = _FW_WM(tmp, CURSORB);
4027         wm->pipe[PIPE_B].primary = _FW_WM_VLV(tmp, PLANEB);
4028         wm->pipe[PIPE_A].primary = _FW_WM_VLV(tmp, PLANEA);
4029
4030         tmp = I915_READ(DSPFW2);
4031         wm->pipe[PIPE_A].sprite[1] = _FW_WM_VLV(tmp, SPRITEB);
4032         wm->pipe[PIPE_A].cursor = _FW_WM(tmp, CURSORA);
4033         wm->pipe[PIPE_A].sprite[0] = _FW_WM_VLV(tmp, SPRITEA);
4034
4035         tmp = I915_READ(DSPFW3);
4036         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
4037
4038         if (IS_CHERRYVIEW(dev_priv)) {
4039                 tmp = I915_READ(DSPFW7_CHV);
4040                 wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
4041                 wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
4042
4043                 tmp = I915_READ(DSPFW8_CHV);
4044                 wm->pipe[PIPE_C].sprite[1] = _FW_WM_VLV(tmp, SPRITEF);
4045                 wm->pipe[PIPE_C].sprite[0] = _FW_WM_VLV(tmp, SPRITEE);
4046
4047                 tmp = I915_READ(DSPFW9_CHV);
4048                 wm->pipe[PIPE_C].primary = _FW_WM_VLV(tmp, PLANEC);
4049                 wm->pipe[PIPE_C].cursor = _FW_WM(tmp, CURSORC);
4050
4051                 tmp = I915_READ(DSPHOWM);
4052                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
4053                 wm->pipe[PIPE_C].sprite[1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
4054                 wm->pipe[PIPE_C].sprite[0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
4055                 wm->pipe[PIPE_C].primary |= _FW_WM(tmp, PLANEC_HI) << 8;
4056                 wm->pipe[PIPE_B].sprite[1] |= _FW_WM(tmp, SPRITED_HI) << 8;
4057                 wm->pipe[PIPE_B].sprite[0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
4058                 wm->pipe[PIPE_B].primary |= _FW_WM(tmp, PLANEB_HI) << 8;
4059                 wm->pipe[PIPE_A].sprite[1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
4060                 wm->pipe[PIPE_A].sprite[0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
4061                 wm->pipe[PIPE_A].primary |= _FW_WM(tmp, PLANEA_HI) << 8;
4062         } else {
4063                 tmp = I915_READ(DSPFW7);
4064                 wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
4065                 wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
4066
4067                 tmp = I915_READ(DSPHOWM);
4068                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
4069                 wm->pipe[PIPE_B].sprite[1] |= _FW_WM(tmp, SPRITED_HI) << 8;
4070                 wm->pipe[PIPE_B].sprite[0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
4071                 wm->pipe[PIPE_B].primary |= _FW_WM(tmp, PLANEB_HI) << 8;
4072                 wm->pipe[PIPE_A].sprite[1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
4073                 wm->pipe[PIPE_A].sprite[0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
4074                 wm->pipe[PIPE_A].primary |= _FW_WM(tmp, PLANEA_HI) << 8;
4075         }
4076 }
4077
4078 #undef _FW_WM
4079 #undef _FW_WM_VLV
4080
4081 void vlv_wm_get_hw_state(struct drm_device *dev)
4082 {
4083         struct drm_i915_private *dev_priv = to_i915(dev);
4084         struct vlv_wm_values *wm = &dev_priv->wm.vlv;
4085         struct intel_plane *plane;
4086         enum pipe pipe;
4087         u32 val;
4088
4089         vlv_read_wm_values(dev_priv, wm);
4090
4091         for_each_intel_plane(dev, plane) {
4092                 switch (plane->base.type) {
4093                         int sprite;
4094                 case DRM_PLANE_TYPE_CURSOR:
4095                         plane->wm.fifo_size = 63;
4096                         break;
4097                 case DRM_PLANE_TYPE_PRIMARY:
4098                         plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, 0);
4099                         break;
4100                 case DRM_PLANE_TYPE_OVERLAY:
4101                         sprite = plane->plane;
4102                         plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, sprite + 1);
4103                         break;
4104                 }
4105         }
4106
4107         wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
4108         wm->level = VLV_WM_LEVEL_PM2;
4109
4110         if (IS_CHERRYVIEW(dev_priv)) {
4111                 mutex_lock(&dev_priv->rps.hw_lock);
4112
4113                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
4114                 if (val & DSP_MAXFIFO_PM5_ENABLE)
4115                         wm->level = VLV_WM_LEVEL_PM5;
4116
4117                 /*
4118                  * If DDR DVFS is disabled in the BIOS, Punit
4119                  * will never ack the request. So if that happens
4120                  * assume we don't have to enable/disable DDR DVFS
4121                  * dynamically. To test that just set the REQ_ACK
4122                  * bit to poke the Punit, but don't change the
4123                  * HIGH/LOW bits so that we don't actually change
4124                  * the current state.
4125                  */
4126                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
4127                 val |= FORCE_DDR_FREQ_REQ_ACK;
4128                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
4129
4130                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
4131                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
4132                         DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
4133                                       "assuming DDR DVFS is disabled\n");
4134                         dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
4135                 } else {
4136                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
4137                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
4138                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
4139                 }
4140
4141                 mutex_unlock(&dev_priv->rps.hw_lock);
4142         }
4143
4144         for_each_pipe(dev_priv, pipe)
4145                 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
4146                               pipe_name(pipe), wm->pipe[pipe].primary, wm->pipe[pipe].cursor,
4147                               wm->pipe[pipe].sprite[0], wm->pipe[pipe].sprite[1]);
4148
4149         DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
4150                       wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
4151 }
4152
4153 void ilk_wm_get_hw_state(struct drm_device *dev)
4154 {
4155         struct drm_i915_private *dev_priv = dev->dev_private;
4156         struct ilk_wm_values *hw = &dev_priv->wm.hw;
4157         struct drm_crtc *crtc;
4158
4159         for_each_crtc(dev, crtc)
4160                 ilk_pipe_wm_get_hw_state(crtc);
4161
4162         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
4163         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
4164         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
4165
4166         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
4167         if (INTEL_INFO(dev)->gen >= 7) {
4168                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
4169                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
4170         }
4171
4172         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
4173                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
4174                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4175         else if (IS_IVYBRIDGE(dev))
4176                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
4177                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4178
4179         hw->enable_fbc_wm =
4180                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4181 }
4182
4183 /**
4184  * intel_update_watermarks - update FIFO watermark values based on current modes
4185  *
4186  * Calculate watermark values for the various WM regs based on current mode
4187  * and plane configuration.
4188  *
4189  * There are several cases to deal with here:
4190  *   - normal (i.e. non-self-refresh)
4191  *   - self-refresh (SR) mode
4192  *   - lines are large relative to FIFO size (buffer can hold up to 2)
4193  *   - lines are small relative to FIFO size (buffer can hold more than 2
4194  *     lines), so need to account for TLB latency
4195  *
4196  *   The normal calculation is:
4197  *     watermark = dotclock * bytes per pixel * latency
4198  *   where latency is platform & configuration dependent (we assume pessimal
4199  *   values here).
4200  *
4201  *   The SR calculation is:
4202  *     watermark = (trunc(latency/line time)+1) * surface width *
4203  *       bytes per pixel
4204  *   where
4205  *     line time = htotal / dotclock
4206  *     surface width = hdisplay for normal plane and 64 for cursor
4207  *   and latency is assumed to be high, as above.
4208  *
4209  * The final value programmed to the register should always be rounded up,
4210  * and include an extra 2 entries to account for clock crossings.
4211  *
4212  * We don't use the sprite, so we can ignore that.  And on Crestline we have
4213  * to set the non-SR watermarks to 8.
4214  */
4215 void intel_update_watermarks(struct drm_crtc *crtc)
4216 {
4217         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
4218
4219         if (dev_priv->display.update_wm)
4220                 dev_priv->display.update_wm(crtc);
4221 }
4222
4223 void intel_update_sprite_watermarks(struct drm_plane *plane,
4224                                     struct drm_crtc *crtc,
4225                                     uint32_t sprite_width,
4226                                     uint32_t sprite_height,
4227                                     int pixel_size,
4228                                     bool enabled, bool scaled)
4229 {
4230         struct drm_i915_private *dev_priv = plane->dev->dev_private;
4231
4232         if (dev_priv->display.update_sprite_wm)
4233                 dev_priv->display.update_sprite_wm(plane, crtc,
4234                                                    sprite_width, sprite_height,
4235                                                    pixel_size, enabled, scaled);
4236 }
4237
4238 /**
4239  * Lock protecting IPS related data structures
4240  */
4241 DEFINE_SPINLOCK(mchdev_lock);
4242
4243 /* Global for IPS driver to get at the current i915 device. Protected by
4244  * mchdev_lock. */
4245 static struct drm_i915_private *i915_mch_dev;
4246
4247 bool ironlake_set_drps(struct drm_device *dev, u8 val)
4248 {
4249         struct drm_i915_private *dev_priv = dev->dev_private;
4250         u16 rgvswctl;
4251
4252         assert_spin_locked(&mchdev_lock);
4253
4254         rgvswctl = I915_READ16(MEMSWCTL);
4255         if (rgvswctl & MEMCTL_CMD_STS) {
4256                 DRM_DEBUG("gpu busy, RCS change rejected\n");
4257                 return false; /* still busy with another command */
4258         }
4259
4260         rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
4261                 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
4262         I915_WRITE16(MEMSWCTL, rgvswctl);
4263         POSTING_READ16(MEMSWCTL);
4264
4265         rgvswctl |= MEMCTL_CMD_STS;
4266         I915_WRITE16(MEMSWCTL, rgvswctl);
4267
4268         return true;
4269 }
4270
4271 static void ironlake_enable_drps(struct drm_device *dev)
4272 {
4273         struct drm_i915_private *dev_priv = dev->dev_private;
4274         u32 rgvmodectl = I915_READ(MEMMODECTL);
4275         u8 fmax, fmin, fstart, vstart;
4276
4277         spin_lock_irq(&mchdev_lock);
4278
4279         /* Enable temp reporting */
4280         I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
4281         I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
4282
4283         /* 100ms RC evaluation intervals */
4284         I915_WRITE(RCUPEI, 100000);
4285         I915_WRITE(RCDNEI, 100000);
4286
4287         /* Set max/min thresholds to 90ms and 80ms respectively */
4288         I915_WRITE(RCBMAXAVG, 90000);
4289         I915_WRITE(RCBMINAVG, 80000);
4290
4291         I915_WRITE(MEMIHYST, 1);
4292
4293         /* Set up min, max, and cur for interrupt handling */
4294         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
4295         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
4296         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
4297                 MEMMODE_FSTART_SHIFT;
4298
4299         vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
4300                 PXVFREQ_PX_SHIFT;
4301
4302         dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
4303         dev_priv->ips.fstart = fstart;
4304
4305         dev_priv->ips.max_delay = fstart;
4306         dev_priv->ips.min_delay = fmin;
4307         dev_priv->ips.cur_delay = fstart;
4308
4309         DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
4310                          fmax, fmin, fstart);
4311
4312         I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
4313
4314         /*
4315          * Interrupts will be enabled in ironlake_irq_postinstall
4316          */
4317
4318         I915_WRITE(VIDSTART, vstart);
4319         POSTING_READ(VIDSTART);
4320
4321         rgvmodectl |= MEMMODE_SWMODE_EN;
4322         I915_WRITE(MEMMODECTL, rgvmodectl);
4323
4324         if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
4325                 DRM_ERROR("stuck trying to change perf mode\n");
4326         mdelay(1);
4327
4328         ironlake_set_drps(dev, fstart);
4329
4330         dev_priv->ips.last_count1 = I915_READ(DMIEC) +
4331                 I915_READ(DDREC) + I915_READ(CSIEC);
4332         dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
4333         dev_priv->ips.last_count2 = I915_READ(GFXEC);
4334         dev_priv->ips.last_time2 = ktime_get_raw_ns();
4335
4336         spin_unlock_irq(&mchdev_lock);
4337 }
4338
4339 static void ironlake_disable_drps(struct drm_device *dev)
4340 {
4341         struct drm_i915_private *dev_priv = dev->dev_private;
4342         u16 rgvswctl;
4343
4344         spin_lock_irq(&mchdev_lock);
4345
4346         rgvswctl = I915_READ16(MEMSWCTL);
4347
4348         /* Ack interrupts, disable EFC interrupt */
4349         I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
4350         I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
4351         I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
4352         I915_WRITE(DEIIR, DE_PCU_EVENT);
4353         I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
4354
4355         /* Go back to the starting frequency */
4356         ironlake_set_drps(dev, dev_priv->ips.fstart);
4357         mdelay(1);
4358         rgvswctl |= MEMCTL_CMD_STS;
4359         I915_WRITE(MEMSWCTL, rgvswctl);
4360         mdelay(1);
4361
4362         spin_unlock_irq(&mchdev_lock);
4363 }
4364
4365 /* There's a funny hw issue where the hw returns all 0 when reading from
4366  * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
4367  * ourselves, instead of doing a rmw cycle (which might result in us clearing
4368  * all limits and the gpu stuck at whatever frequency it is at atm).
4369  */
4370 static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
4371 {
4372         u32 limits;
4373
4374         /* Only set the down limit when we've reached the lowest level to avoid
4375          * getting more interrupts, otherwise leave this clear. This prevents a
4376          * race in the hw when coming out of rc6: There's a tiny window where
4377          * the hw runs at the minimal clock before selecting the desired
4378          * frequency, if the down threshold expires in that window we will not
4379          * receive a down interrupt. */
4380         if (IS_GEN9(dev_priv->dev)) {
4381                 limits = (dev_priv->rps.max_freq_softlimit) << 23;
4382                 if (val <= dev_priv->rps.min_freq_softlimit)
4383                         limits |= (dev_priv->rps.min_freq_softlimit) << 14;
4384         } else {
4385                 limits = dev_priv->rps.max_freq_softlimit << 24;
4386                 if (val <= dev_priv->rps.min_freq_softlimit)
4387                         limits |= dev_priv->rps.min_freq_softlimit << 16;
4388         }
4389
4390         return limits;
4391 }
4392
4393 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
4394 {
4395         int new_power;
4396         u32 threshold_up = 0, threshold_down = 0; /* in % */
4397         u32 ei_up = 0, ei_down = 0;
4398
4399         new_power = dev_priv->rps.power;
4400         switch (dev_priv->rps.power) {
4401         case LOW_POWER:
4402                 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
4403                         new_power = BETWEEN;
4404                 break;
4405
4406         case BETWEEN:
4407                 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
4408                         new_power = LOW_POWER;
4409                 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
4410                         new_power = HIGH_POWER;
4411                 break;
4412
4413         case HIGH_POWER:
4414                 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
4415                         new_power = BETWEEN;
4416                 break;
4417         }
4418         /* Max/min bins are special */
4419         if (val <= dev_priv->rps.min_freq_softlimit)
4420                 new_power = LOW_POWER;
4421         if (val >= dev_priv->rps.max_freq_softlimit)
4422                 new_power = HIGH_POWER;
4423         if (new_power == dev_priv->rps.power)
4424                 return;
4425
4426         /* Note the units here are not exactly 1us, but 1280ns. */
4427         switch (new_power) {
4428         case LOW_POWER:
4429                 /* Upclock if more than 95% busy over 16ms */
4430                 ei_up = 16000;
4431                 threshold_up = 95;
4432
4433                 /* Downclock if less than 85% busy over 32ms */
4434                 ei_down = 32000;
4435                 threshold_down = 85;
4436                 break;
4437
4438         case BETWEEN:
4439                 /* Upclock if more than 90% busy over 13ms */
4440                 ei_up = 13000;
4441                 threshold_up = 90;
4442
4443                 /* Downclock if less than 75% busy over 32ms */
4444                 ei_down = 32000;
4445                 threshold_down = 75;
4446                 break;
4447
4448         case HIGH_POWER:
4449                 /* Upclock if more than 85% busy over 10ms */
4450                 ei_up = 10000;
4451                 threshold_up = 85;
4452
4453                 /* Downclock if less than 60% busy over 32ms */
4454                 ei_down = 32000;
4455                 threshold_down = 60;
4456                 break;
4457         }
4458
4459         I915_WRITE(GEN6_RP_UP_EI,
4460                 GT_INTERVAL_FROM_US(dev_priv, ei_up));
4461         I915_WRITE(GEN6_RP_UP_THRESHOLD,
4462                 GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100)));
4463
4464         I915_WRITE(GEN6_RP_DOWN_EI,
4465                 GT_INTERVAL_FROM_US(dev_priv, ei_down));
4466         I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
4467                 GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100)));
4468
4469          I915_WRITE(GEN6_RP_CONTROL,
4470                     GEN6_RP_MEDIA_TURBO |
4471                     GEN6_RP_MEDIA_HW_NORMAL_MODE |
4472                     GEN6_RP_MEDIA_IS_GFX |
4473                     GEN6_RP_ENABLE |
4474                     GEN6_RP_UP_BUSY_AVG |
4475                     GEN6_RP_DOWN_IDLE_AVG);
4476
4477         dev_priv->rps.power = new_power;
4478         dev_priv->rps.up_threshold = threshold_up;
4479         dev_priv->rps.down_threshold = threshold_down;
4480         dev_priv->rps.last_adj = 0;
4481 }
4482
4483 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
4484 {
4485         u32 mask = 0;
4486
4487         if (val > dev_priv->rps.min_freq_softlimit)
4488                 mask |= GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
4489         if (val < dev_priv->rps.max_freq_softlimit)
4490                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
4491
4492         mask &= dev_priv->pm_rps_events;
4493
4494         return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
4495 }
4496
4497 /* gen6_set_rps is called to update the frequency request, but should also be
4498  * called when the range (min_delay and max_delay) is modified so that we can
4499  * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
4500 static void gen6_set_rps(struct drm_device *dev, u8 val)
4501 {
4502         struct drm_i915_private *dev_priv = dev->dev_private;
4503
4504         /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
4505         if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0))
4506                 return;
4507
4508         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4509         WARN_ON(val > dev_priv->rps.max_freq);
4510         WARN_ON(val < dev_priv->rps.min_freq);
4511
4512         /* min/max delay may still have been modified so be sure to
4513          * write the limits value.
4514          */
4515         if (val != dev_priv->rps.cur_freq) {
4516                 gen6_set_rps_thresholds(dev_priv, val);
4517
4518                 if (IS_GEN9(dev))
4519                         I915_WRITE(GEN6_RPNSWREQ,
4520                                    GEN9_FREQUENCY(val));
4521                 else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
4522                         I915_WRITE(GEN6_RPNSWREQ,
4523                                    HSW_FREQUENCY(val));
4524                 else
4525                         I915_WRITE(GEN6_RPNSWREQ,
4526                                    GEN6_FREQUENCY(val) |
4527                                    GEN6_OFFSET(0) |
4528                                    GEN6_AGGRESSIVE_TURBO);
4529         }
4530
4531         /* Make sure we continue to get interrupts
4532          * until we hit the minimum or maximum frequencies.
4533          */
4534         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
4535         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
4536
4537         POSTING_READ(GEN6_RPNSWREQ);
4538
4539         dev_priv->rps.cur_freq = val;
4540         trace_intel_gpu_freq_change(val * 50);
4541 }
4542
4543 static void valleyview_set_rps(struct drm_device *dev, u8 val)
4544 {
4545         struct drm_i915_private *dev_priv = dev->dev_private;
4546
4547         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4548         WARN_ON(val > dev_priv->rps.max_freq);
4549         WARN_ON(val < dev_priv->rps.min_freq);
4550
4551         if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
4552                       "Odd GPU freq value\n"))
4553                 val &= ~1;
4554
4555         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
4556
4557         if (val != dev_priv->rps.cur_freq) {
4558                 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
4559                 if (!IS_CHERRYVIEW(dev_priv))
4560                         gen6_set_rps_thresholds(dev_priv, val);
4561         }
4562
4563         dev_priv->rps.cur_freq = val;
4564         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
4565 }
4566
4567 /* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
4568  *
4569  * * If Gfx is Idle, then
4570  * 1. Forcewake Media well.
4571  * 2. Request idle freq.
4572  * 3. Release Forcewake of Media well.
4573 */
4574 static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
4575 {
4576         u32 val = dev_priv->rps.idle_freq;
4577
4578         if (dev_priv->rps.cur_freq <= val)
4579                 return;
4580
4581         /* Wake up the media well, as that takes a lot less
4582          * power than the Render well. */
4583         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
4584         valleyview_set_rps(dev_priv->dev, val);
4585         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
4586 }
4587
4588 void gen6_rps_busy(struct drm_i915_private *dev_priv)
4589 {
4590         mutex_lock(&dev_priv->rps.hw_lock);
4591         if (dev_priv->rps.enabled) {
4592                 if (dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED))
4593                         gen6_rps_reset_ei(dev_priv);
4594                 I915_WRITE(GEN6_PMINTRMSK,
4595                            gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
4596         }
4597         mutex_unlock(&dev_priv->rps.hw_lock);
4598 }
4599
4600 void gen6_rps_idle(struct drm_i915_private *dev_priv)
4601 {
4602         struct drm_device *dev = dev_priv->dev;
4603
4604         mutex_lock(&dev_priv->rps.hw_lock);
4605         if (dev_priv->rps.enabled) {
4606                 if (IS_VALLEYVIEW(dev))
4607                         vlv_set_rps_idle(dev_priv);
4608                 else
4609                         gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4610                 dev_priv->rps.last_adj = 0;
4611                 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
4612         }
4613         mutex_unlock(&dev_priv->rps.hw_lock);
4614
4615         spin_lock(&dev_priv->rps.client_lock);
4616         while (!list_empty(&dev_priv->rps.clients))
4617                 list_del_init(dev_priv->rps.clients.next);
4618         spin_unlock(&dev_priv->rps.client_lock);
4619 }
4620
4621 void gen6_rps_boost(struct drm_i915_private *dev_priv,
4622                     struct intel_rps_client *rps,
4623                     unsigned long submitted)
4624 {
4625         /* This is intentionally racy! We peek at the state here, then
4626          * validate inside the RPS worker.
4627          */
4628         if (!(dev_priv->mm.busy &&
4629               dev_priv->rps.enabled &&
4630               dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit))
4631                 return;
4632
4633         /* Force a RPS boost (and don't count it against the client) if
4634          * the GPU is severely congested.
4635          */
4636         if (rps && time_after(jiffies, submitted + DRM_I915_THROTTLE_JIFFIES))
4637                 rps = NULL;
4638
4639         spin_lock(&dev_priv->rps.client_lock);
4640         if (rps == NULL || list_empty(&rps->link)) {
4641                 spin_lock_irq(&dev_priv->irq_lock);
4642                 if (dev_priv->rps.interrupts_enabled) {
4643                         dev_priv->rps.client_boost = true;
4644                         queue_work(dev_priv->wq, &dev_priv->rps.work);
4645                 }
4646                 spin_unlock_irq(&dev_priv->irq_lock);
4647
4648                 if (rps != NULL) {
4649                         list_add(&rps->link, &dev_priv->rps.clients);
4650                         rps->boosts++;
4651                 } else
4652                         dev_priv->rps.boosts++;
4653         }
4654         spin_unlock(&dev_priv->rps.client_lock);
4655 }
4656
4657 void intel_set_rps(struct drm_device *dev, u8 val)
4658 {
4659         if (IS_VALLEYVIEW(dev))
4660                 valleyview_set_rps(dev, val);
4661         else
4662                 gen6_set_rps(dev, val);
4663 }
4664
4665 static void gen9_disable_rps(struct drm_device *dev)
4666 {
4667         struct drm_i915_private *dev_priv = dev->dev_private;
4668
4669         I915_WRITE(GEN6_RC_CONTROL, 0);
4670         I915_WRITE(GEN9_PG_ENABLE, 0);
4671 }
4672
4673 static void gen6_disable_rps(struct drm_device *dev)
4674 {
4675         struct drm_i915_private *dev_priv = dev->dev_private;
4676
4677         I915_WRITE(GEN6_RC_CONTROL, 0);
4678         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
4679 }
4680
4681 static void cherryview_disable_rps(struct drm_device *dev)
4682 {
4683         struct drm_i915_private *dev_priv = dev->dev_private;
4684
4685         I915_WRITE(GEN6_RC_CONTROL, 0);
4686 }
4687
4688 static void valleyview_disable_rps(struct drm_device *dev)
4689 {
4690         struct drm_i915_private *dev_priv = dev->dev_private;
4691
4692         /* we're doing forcewake before Disabling RC6,
4693          * This what the BIOS expects when going into suspend */
4694         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4695
4696         I915_WRITE(GEN6_RC_CONTROL, 0);
4697
4698         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4699 }
4700
4701 static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
4702 {
4703         if (IS_VALLEYVIEW(dev)) {
4704                 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1)))
4705                         mode = GEN6_RC_CTL_RC6_ENABLE;
4706                 else
4707                         mode = 0;
4708         }
4709         if (HAS_RC6p(dev))
4710                 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
4711                               (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
4712                               (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
4713                               (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
4714
4715         else
4716                 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
4717                               (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
4718 }
4719
4720 static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
4721 {
4722         /* No RC6 before Ironlake and code is gone for ilk. */
4723         if (INTEL_INFO(dev)->gen < 6)
4724                 return 0;
4725
4726         /* Respect the kernel parameter if it is set */
4727         if (enable_rc6 >= 0) {
4728                 int mask;
4729
4730                 if (HAS_RC6p(dev))
4731                         mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE |
4732                                INTEL_RC6pp_ENABLE;
4733                 else
4734                         mask = INTEL_RC6_ENABLE;
4735
4736                 if ((enable_rc6 & mask) != enable_rc6)
4737                         DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
4738                                       enable_rc6 & mask, enable_rc6, mask);
4739
4740                 return enable_rc6 & mask;
4741         }
4742
4743         if (IS_IVYBRIDGE(dev))
4744                 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
4745
4746         return INTEL_RC6_ENABLE;
4747 }
4748
4749 int intel_enable_rc6(const struct drm_device *dev)
4750 {
4751         return i915.enable_rc6;
4752 }
4753
4754 static void gen6_init_rps_frequencies(struct drm_device *dev)
4755 {
4756         struct drm_i915_private *dev_priv = dev->dev_private;
4757         uint32_t rp_state_cap;
4758         u32 ddcc_status = 0;
4759         int ret;
4760
4761         /* All of these values are in units of 50MHz */
4762         dev_priv->rps.cur_freq          = 0;
4763         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
4764         if (IS_BROXTON(dev)) {
4765                 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
4766                 dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff;
4767                 dev_priv->rps.rp1_freq = (rp_state_cap >>  8) & 0xff;
4768                 dev_priv->rps.min_freq = (rp_state_cap >>  0) & 0xff;
4769         } else {
4770                 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
4771                 dev_priv->rps.rp0_freq = (rp_state_cap >>  0) & 0xff;
4772                 dev_priv->rps.rp1_freq = (rp_state_cap >>  8) & 0xff;
4773                 dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
4774         }
4775
4776         /* hw_max = RP0 until we check for overclocking */
4777         dev_priv->rps.max_freq          = dev_priv->rps.rp0_freq;
4778
4779         dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
4780         if (IS_HASWELL(dev) || IS_BROADWELL(dev) || IS_SKYLAKE(dev)) {
4781                 ret = sandybridge_pcode_read(dev_priv,
4782                                         HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
4783                                         &ddcc_status);
4784                 if (0 == ret)
4785                         dev_priv->rps.efficient_freq =
4786                                 clamp_t(u8,
4787                                         ((ddcc_status >> 8) & 0xff),
4788                                         dev_priv->rps.min_freq,
4789                                         dev_priv->rps.max_freq);
4790         }
4791
4792         if (IS_SKYLAKE(dev)) {
4793                 /* Store the frequency values in 16.66 MHZ units, which is
4794                    the natural hardware unit for SKL */
4795                 dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
4796                 dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER;
4797                 dev_priv->rps.min_freq *= GEN9_FREQ_SCALER;
4798                 dev_priv->rps.max_freq *= GEN9_FREQ_SCALER;
4799                 dev_priv->rps.efficient_freq *= GEN9_FREQ_SCALER;
4800         }
4801
4802         dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
4803
4804         /* Preserve min/max settings in case of re-init */
4805         if (dev_priv->rps.max_freq_softlimit == 0)
4806                 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4807
4808         if (dev_priv->rps.min_freq_softlimit == 0) {
4809                 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
4810                         dev_priv->rps.min_freq_softlimit =
4811                                 max_t(int, dev_priv->rps.efficient_freq,
4812                                       intel_freq_opcode(dev_priv, 450));
4813                 else
4814                         dev_priv->rps.min_freq_softlimit =
4815                                 dev_priv->rps.min_freq;
4816         }
4817 }
4818
4819 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
4820 static void gen9_enable_rps(struct drm_device *dev)
4821 {
4822         struct drm_i915_private *dev_priv = dev->dev_private;
4823
4824         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4825
4826         gen6_init_rps_frequencies(dev);
4827
4828         /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
4829         if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) {
4830                 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4831                 return;
4832         }
4833
4834         /* Program defaults and thresholds for RPS*/
4835         I915_WRITE(GEN6_RC_VIDEO_FREQ,
4836                 GEN9_FREQUENCY(dev_priv->rps.rp1_freq));
4837
4838         /* 1 second timeout*/
4839         I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
4840                 GT_INTERVAL_FROM_US(dev_priv, 1000000));
4841
4842         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
4843
4844         /* Leaning on the below call to gen6_set_rps to program/setup the
4845          * Up/Down EI & threshold registers, as well as the RP_CONTROL,
4846          * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
4847         dev_priv->rps.power = HIGH_POWER; /* force a reset */
4848         gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4849
4850         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4851 }
4852
4853 static void gen9_enable_rc6(struct drm_device *dev)
4854 {
4855         struct drm_i915_private *dev_priv = dev->dev_private;
4856         struct intel_engine_cs *ring;
4857         uint32_t rc6_mask = 0;
4858         int unused;
4859
4860         /* 1a: Software RC state - RC0 */
4861         I915_WRITE(GEN6_RC_STATE, 0);
4862
4863         /* 1b: Get forcewake during program sequence. Although the driver
4864          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
4865         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4866
4867         /* 2a: Disable RC states. */
4868         I915_WRITE(GEN6_RC_CONTROL, 0);
4869
4870         /* 2b: Program RC6 thresholds.*/
4871
4872         /* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */
4873         if (IS_SKYLAKE(dev) && !((IS_SKL_GT3(dev) || IS_SKL_GT4(dev)) &&
4874                                  (INTEL_REVID(dev) <= SKL_REVID_E0)))
4875                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
4876         else
4877                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
4878         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4879         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4880         for_each_ring(ring, dev_priv, unused)
4881                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4882
4883         if (HAS_GUC_UCODE(dev))
4884                 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
4885
4886         I915_WRITE(GEN6_RC_SLEEP, 0);
4887         I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
4888
4889         /* 2c: Program Coarse Power Gating Policies. */
4890         I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
4891         I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
4892
4893         /* 3a: Enable RC6 */
4894         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4895                 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
4896         DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
4897                         "on" : "off");
4898
4899         if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_D0) ||
4900             (IS_BROXTON(dev) && INTEL_REVID(dev) <= BXT_REVID_A0))
4901                 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4902                            GEN7_RC_CTL_TO_MODE |
4903                            rc6_mask);
4904         else
4905                 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4906                            GEN6_RC_CTL_EI_MODE(1) |
4907                            rc6_mask);
4908
4909         /*
4910          * 3b: Enable Coarse Power Gating only when RC6 is enabled.
4911          * WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6.
4912          */
4913         if ((IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) ||
4914             ((IS_SKL_GT3(dev) || IS_SKL_GT4(dev)) && (INTEL_REVID(dev) <= SKL_REVID_E0)))
4915                 I915_WRITE(GEN9_PG_ENABLE, 0);
4916         else
4917                 I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
4918                                 (GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE) : 0);
4919
4920         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4921
4922 }
4923
4924 static void gen8_enable_rps(struct drm_device *dev)
4925 {
4926         struct drm_i915_private *dev_priv = dev->dev_private;
4927         struct intel_engine_cs *ring;
4928         uint32_t rc6_mask = 0;
4929         int unused;
4930
4931         /* 1a: Software RC state - RC0 */
4932         I915_WRITE(GEN6_RC_STATE, 0);
4933
4934         /* 1c & 1d: Get forcewake during program sequence. Although the driver
4935          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
4936         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4937
4938         /* 2a: Disable RC states. */
4939         I915_WRITE(GEN6_RC_CONTROL, 0);
4940
4941         /* Initialize rps frequencies */
4942         gen6_init_rps_frequencies(dev);
4943
4944         /* 2b: Program RC6 thresholds.*/
4945         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
4946         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4947         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4948         for_each_ring(ring, dev_priv, unused)
4949                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4950         I915_WRITE(GEN6_RC_SLEEP, 0);
4951         if (IS_BROADWELL(dev))
4952                 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
4953         else
4954                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
4955
4956         /* 3: Enable RC6 */
4957         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4958                 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
4959         intel_print_rc6_info(dev, rc6_mask);
4960         if (IS_BROADWELL(dev))
4961                 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4962                                 GEN7_RC_CTL_TO_MODE |
4963                                 rc6_mask);
4964         else
4965                 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4966                                 GEN6_RC_CTL_EI_MODE(1) |
4967                                 rc6_mask);
4968
4969         /* 4 Program defaults and thresholds for RPS*/
4970         I915_WRITE(GEN6_RPNSWREQ,
4971                    HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4972         I915_WRITE(GEN6_RC_VIDEO_FREQ,
4973                    HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4974         /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
4975         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
4976
4977         /* Docs recommend 900MHz, and 300 MHz respectively */
4978         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
4979                    dev_priv->rps.max_freq_softlimit << 24 |
4980                    dev_priv->rps.min_freq_softlimit << 16);
4981
4982         I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
4983         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
4984         I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
4985         I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
4986
4987         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4988
4989         /* 5: Enable RPS */
4990         I915_WRITE(GEN6_RP_CONTROL,
4991                    GEN6_RP_MEDIA_TURBO |
4992                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
4993                    GEN6_RP_MEDIA_IS_GFX |
4994                    GEN6_RP_ENABLE |
4995                    GEN6_RP_UP_BUSY_AVG |
4996                    GEN6_RP_DOWN_IDLE_AVG);
4997
4998         /* 6: Ring frequency + overclocking (our driver does this later */
4999
5000         dev_priv->rps.power = HIGH_POWER; /* force a reset */
5001         gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
5002
5003         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5004 }
5005
5006 static void gen6_enable_rps(struct drm_device *dev)
5007 {
5008         struct drm_i915_private *dev_priv = dev->dev_private;
5009         struct intel_engine_cs *ring;
5010         u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
5011         u32 gtfifodbg;
5012         int rc6_mode;
5013         int i, ret;
5014
5015         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5016
5017         /* Here begins a magic sequence of register writes to enable
5018          * auto-downclocking.
5019          *
5020          * Perhaps there might be some value in exposing these to
5021          * userspace...
5022          */
5023         I915_WRITE(GEN6_RC_STATE, 0);
5024
5025         /* Clear the DBG now so we don't confuse earlier errors */
5026         if ((gtfifodbg = I915_READ(GTFIFODBG))) {
5027                 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
5028                 I915_WRITE(GTFIFODBG, gtfifodbg);
5029         }
5030
5031         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5032
5033         /* Initialize rps frequencies */
5034         gen6_init_rps_frequencies(dev);
5035
5036         /* disable the counters and set deterministic thresholds */
5037         I915_WRITE(GEN6_RC_CONTROL, 0);
5038
5039         I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
5040         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
5041         I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
5042         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
5043         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
5044
5045         for_each_ring(ring, dev_priv, i)
5046                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
5047
5048         I915_WRITE(GEN6_RC_SLEEP, 0);
5049         I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
5050         if (IS_IVYBRIDGE(dev))
5051                 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
5052         else
5053                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
5054         I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
5055         I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
5056
5057         /* Check if we are enabling RC6 */
5058         rc6_mode = intel_enable_rc6(dev_priv->dev);
5059         if (rc6_mode & INTEL_RC6_ENABLE)
5060                 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
5061
5062         /* We don't use those on Haswell */
5063         if (!IS_HASWELL(dev)) {
5064                 if (rc6_mode & INTEL_RC6p_ENABLE)
5065                         rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
5066
5067                 if (rc6_mode & INTEL_RC6pp_ENABLE)
5068                         rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
5069         }
5070
5071         intel_print_rc6_info(dev, rc6_mask);
5072
5073         I915_WRITE(GEN6_RC_CONTROL,
5074                    rc6_mask |
5075                    GEN6_RC_CTL_EI_MODE(1) |
5076                    GEN6_RC_CTL_HW_ENABLE);
5077
5078         /* Power down if completely idle for over 50ms */
5079         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
5080         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
5081
5082         ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
5083         if (ret)
5084                 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
5085
5086         ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
5087         if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
5088                 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
5089                                  (dev_priv->rps.max_freq_softlimit & 0xff) * 50,
5090                                  (pcu_mbox & 0xff) * 50);
5091                 dev_priv->rps.max_freq = pcu_mbox & 0xff;
5092         }
5093
5094         dev_priv->rps.power = HIGH_POWER; /* force a reset */
5095         gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
5096
5097         rc6vids = 0;
5098         ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
5099         if (IS_GEN6(dev) && ret) {
5100                 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
5101         } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
5102                 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
5103                           GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
5104                 rc6vids &= 0xffff00;
5105                 rc6vids |= GEN6_ENCODE_RC6_VID(450);
5106                 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
5107                 if (ret)
5108                         DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
5109         }
5110
5111         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5112 }
5113
5114 static void __gen6_update_ring_freq(struct drm_device *dev)
5115 {
5116         struct drm_i915_private *dev_priv = dev->dev_private;
5117         int min_freq = 15;
5118         unsigned int gpu_freq;
5119         unsigned int max_ia_freq, min_ring_freq;
5120         unsigned int max_gpu_freq, min_gpu_freq;
5121         int scaling_factor = 180;
5122         struct cpufreq_policy *policy;
5123
5124         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5125
5126         policy = cpufreq_cpu_get(0);
5127         if (policy) {
5128                 max_ia_freq = policy->cpuinfo.max_freq;
5129                 cpufreq_cpu_put(policy);
5130         } else {
5131                 /*
5132                  * Default to measured freq if none found, PCU will ensure we
5133                  * don't go over
5134                  */
5135                 max_ia_freq = tsc_khz;
5136         }
5137
5138         /* Convert from kHz to MHz */
5139         max_ia_freq /= 1000;
5140
5141         min_ring_freq = I915_READ(DCLK) & 0xf;
5142         /* convert DDR frequency from units of 266.6MHz to bandwidth */
5143         min_ring_freq = mult_frac(min_ring_freq, 8, 3);
5144
5145         if (IS_SKYLAKE(dev)) {
5146                 /* Convert GT frequency to 50 HZ units */
5147                 min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
5148                 max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
5149         } else {
5150                 min_gpu_freq = dev_priv->rps.min_freq;
5151                 max_gpu_freq = dev_priv->rps.max_freq;
5152         }
5153
5154         /*
5155          * For each potential GPU frequency, load a ring frequency we'd like
5156          * to use for memory access.  We do this by specifying the IA frequency
5157          * the PCU should use as a reference to determine the ring frequency.
5158          */
5159         for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
5160                 int diff = max_gpu_freq - gpu_freq;
5161                 unsigned int ia_freq = 0, ring_freq = 0;
5162
5163                 if (IS_SKYLAKE(dev)) {
5164                         /*
5165                          * ring_freq = 2 * GT. ring_freq is in 100MHz units
5166                          * No floor required for ring frequency on SKL.
5167                          */
5168                         ring_freq = gpu_freq;
5169                 } else if (INTEL_INFO(dev)->gen >= 8) {
5170                         /* max(2 * GT, DDR). NB: GT is 50MHz units */
5171                         ring_freq = max(min_ring_freq, gpu_freq);
5172                 } else if (IS_HASWELL(dev)) {
5173                         ring_freq = mult_frac(gpu_freq, 5, 4);
5174                         ring_freq = max(min_ring_freq, ring_freq);
5175                         /* leave ia_freq as the default, chosen by cpufreq */
5176                 } else {
5177                         /* On older processors, there is no separate ring
5178                          * clock domain, so in order to boost the bandwidth
5179                          * of the ring, we need to upclock the CPU (ia_freq).
5180                          *
5181                          * For GPU frequencies less than 750MHz,
5182                          * just use the lowest ring freq.
5183                          */
5184                         if (gpu_freq < min_freq)
5185                                 ia_freq = 800;
5186                         else
5187                                 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
5188                         ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
5189                 }
5190
5191                 sandybridge_pcode_write(dev_priv,
5192                                         GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
5193                                         ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
5194                                         ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
5195                                         gpu_freq);
5196         }
5197 }
5198
5199 void gen6_update_ring_freq(struct drm_device *dev)
5200 {
5201         struct drm_i915_private *dev_priv = dev->dev_private;
5202
5203         if (!HAS_CORE_RING_FREQ(dev))
5204                 return;
5205
5206         mutex_lock(&dev_priv->rps.hw_lock);
5207         __gen6_update_ring_freq(dev);
5208         mutex_unlock(&dev_priv->rps.hw_lock);
5209 }
5210
5211 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
5212 {
5213         struct drm_device *dev = dev_priv->dev;
5214         u32 val, rp0;
5215
5216         if (dev->pdev->revision >= 0x20) {
5217                 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
5218
5219                 switch (INTEL_INFO(dev)->eu_total) {
5220                 case 8:
5221                                 /* (2 * 4) config */
5222                                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
5223                                 break;
5224                 case 12:
5225                                 /* (2 * 6) config */
5226                                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
5227                                 break;
5228                 case 16:
5229                                 /* (2 * 8) config */
5230                 default:
5231                                 /* Setting (2 * 8) Min RP0 for any other combination */
5232                                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
5233                                 break;
5234                 }
5235                 rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
5236         } else {
5237                 /* For pre-production hardware */
5238                 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
5239                 rp0 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) &
5240                        PUNIT_GPU_STATUS_MAX_FREQ_MASK;
5241         }
5242         return rp0;
5243 }
5244
5245 static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
5246 {
5247         u32 val, rpe;
5248
5249         val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
5250         rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
5251
5252         return rpe;
5253 }
5254
5255 static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
5256 {
5257         struct drm_device *dev = dev_priv->dev;
5258         u32 val, rp1;
5259
5260         if (dev->pdev->revision >= 0x20) {
5261                 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
5262                 rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
5263         } else {
5264                 /* For pre-production hardware */
5265                 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
5266                 rp1 = ((val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) &
5267                        PUNIT_GPU_STATUS_MAX_FREQ_MASK);
5268         }
5269         return rp1;
5270 }
5271
5272 static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
5273 {
5274         u32 val, rp1;
5275
5276         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
5277
5278         rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
5279
5280         return rp1;
5281 }
5282
5283 static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
5284 {
5285         u32 val, rp0;
5286
5287         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
5288
5289         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
5290         /* Clamp to max */
5291         rp0 = min_t(u32, rp0, 0xea);
5292
5293         return rp0;
5294 }
5295
5296 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
5297 {
5298         u32 val, rpe;
5299
5300         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
5301         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
5302         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
5303         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
5304
5305         return rpe;
5306 }
5307
5308 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
5309 {
5310         return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
5311 }
5312
5313 /* Check that the pctx buffer wasn't move under us. */
5314 static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
5315 {
5316         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
5317
5318         WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
5319                              dev_priv->vlv_pctx->stolen->start);
5320 }
5321
5322
5323 /* Check that the pcbr address is not empty. */
5324 static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
5325 {
5326         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
5327
5328         WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
5329 }
5330
5331 static void cherryview_setup_pctx(struct drm_device *dev)
5332 {
5333         struct drm_i915_private *dev_priv = dev->dev_private;
5334         unsigned long pctx_paddr, paddr;
5335         struct i915_gtt *gtt = &dev_priv->gtt;
5336         u32 pcbr;
5337         int pctx_size = 32*1024;
5338
5339         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
5340
5341         pcbr = I915_READ(VLV_PCBR);
5342         if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
5343                 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
5344                 paddr = (dev_priv->mm.stolen_base +
5345                          (gtt->stolen_size - pctx_size));
5346
5347                 pctx_paddr = (paddr & (~4095));
5348                 I915_WRITE(VLV_PCBR, pctx_paddr);
5349         }
5350
5351         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5352 }
5353
5354 static void valleyview_setup_pctx(struct drm_device *dev)
5355 {
5356         struct drm_i915_private *dev_priv = dev->dev_private;
5357         struct drm_i915_gem_object *pctx;
5358         unsigned long pctx_paddr;
5359         u32 pcbr;
5360         int pctx_size = 24*1024;
5361
5362         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
5363
5364         pcbr = I915_READ(VLV_PCBR);
5365         if (pcbr) {
5366                 /* BIOS set it up already, grab the pre-alloc'd space */
5367                 int pcbr_offset;
5368
5369                 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
5370                 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
5371                                                                       pcbr_offset,
5372                                                                       I915_GTT_OFFSET_NONE,
5373                                                                       pctx_size);
5374                 goto out;
5375         }
5376
5377         DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
5378
5379         /*
5380          * From the Gunit register HAS:
5381          * The Gfx driver is expected to program this register and ensure
5382          * proper allocation within Gfx stolen memory.  For example, this
5383          * register should be programmed such than the PCBR range does not
5384          * overlap with other ranges, such as the frame buffer, protected
5385          * memory, or any other relevant ranges.
5386          */
5387         pctx = i915_gem_object_create_stolen(dev, pctx_size);
5388         if (!pctx) {
5389                 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
5390                 return;
5391         }
5392
5393         pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
5394         I915_WRITE(VLV_PCBR, pctx_paddr);
5395
5396 out:
5397         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5398         dev_priv->vlv_pctx = pctx;
5399 }
5400
5401 static void valleyview_cleanup_pctx(struct drm_device *dev)
5402 {
5403         struct drm_i915_private *dev_priv = dev->dev_private;
5404
5405         if (WARN_ON(!dev_priv->vlv_pctx))
5406                 return;
5407
5408         drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
5409         dev_priv->vlv_pctx = NULL;
5410 }
5411
5412 static void valleyview_init_gt_powersave(struct drm_device *dev)
5413 {
5414         struct drm_i915_private *dev_priv = dev->dev_private;
5415         u32 val;
5416
5417         valleyview_setup_pctx(dev);
5418
5419         mutex_lock(&dev_priv->rps.hw_lock);
5420
5421         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
5422         switch ((val >> 6) & 3) {
5423         case 0:
5424         case 1:
5425                 dev_priv->mem_freq = 800;
5426                 break;
5427         case 2:
5428                 dev_priv->mem_freq = 1066;
5429                 break;
5430         case 3:
5431                 dev_priv->mem_freq = 1333;
5432                 break;
5433         }
5434         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
5435
5436         dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
5437         dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
5438         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
5439                          intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5440                          dev_priv->rps.max_freq);
5441
5442         dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
5443         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
5444                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5445                          dev_priv->rps.efficient_freq);
5446
5447         dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
5448         DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
5449                          intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5450                          dev_priv->rps.rp1_freq);
5451
5452         dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
5453         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
5454                          intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5455                          dev_priv->rps.min_freq);
5456
5457         dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
5458
5459         /* Preserve min/max settings in case of re-init */
5460         if (dev_priv->rps.max_freq_softlimit == 0)
5461                 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
5462
5463         if (dev_priv->rps.min_freq_softlimit == 0)
5464                 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
5465
5466         mutex_unlock(&dev_priv->rps.hw_lock);
5467 }
5468
5469 static void cherryview_init_gt_powersave(struct drm_device *dev)
5470 {
5471         struct drm_i915_private *dev_priv = dev->dev_private;
5472         u32 val;
5473
5474         cherryview_setup_pctx(dev);
5475
5476         mutex_lock(&dev_priv->rps.hw_lock);
5477
5478         mutex_lock(&dev_priv->sb_lock);
5479         val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
5480         mutex_unlock(&dev_priv->sb_lock);
5481
5482         switch ((val >> 2) & 0x7) {
5483         case 0:
5484         case 1:
5485                 dev_priv->rps.cz_freq = 200;
5486                 dev_priv->mem_freq = 1600;
5487                 break;
5488         case 2:
5489                 dev_priv->rps.cz_freq = 267;
5490                 dev_priv->mem_freq = 1600;
5491                 break;
5492         case 3:
5493                 dev_priv->rps.cz_freq = 333;
5494                 dev_priv->mem_freq = 2000;
5495                 break;
5496         case 4:
5497                 dev_priv->rps.cz_freq = 320;
5498                 dev_priv->mem_freq = 1600;
5499                 break;
5500         case 5:
5501                 dev_priv->rps.cz_freq = 400;
5502                 dev_priv->mem_freq = 1600;
5503                 break;
5504         }
5505         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
5506
5507         dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
5508         dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
5509         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
5510                          intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5511                          dev_priv->rps.max_freq);
5512
5513         dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
5514         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
5515                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5516                          dev_priv->rps.efficient_freq);
5517
5518         dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
5519         DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
5520                          intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5521                          dev_priv->rps.rp1_freq);
5522
5523         /* PUnit validated range is only [RPe, RP0] */
5524         dev_priv->rps.min_freq = dev_priv->rps.efficient_freq;
5525         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
5526                          intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5527                          dev_priv->rps.min_freq);
5528
5529         WARN_ONCE((dev_priv->rps.max_freq |
5530                    dev_priv->rps.efficient_freq |
5531                    dev_priv->rps.rp1_freq |
5532                    dev_priv->rps.min_freq) & 1,
5533                   "Odd GPU freq values\n");
5534
5535         dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
5536
5537         /* Preserve min/max settings in case of re-init */
5538         if (dev_priv->rps.max_freq_softlimit == 0)
5539                 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
5540
5541         if (dev_priv->rps.min_freq_softlimit == 0)
5542                 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
5543
5544         mutex_unlock(&dev_priv->rps.hw_lock);
5545 }
5546
5547 static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
5548 {
5549         valleyview_cleanup_pctx(dev);
5550 }
5551
5552 static void cherryview_enable_rps(struct drm_device *dev)
5553 {
5554         struct drm_i915_private *dev_priv = dev->dev_private;
5555         struct intel_engine_cs *ring;
5556         u32 gtfifodbg, val, rc6_mode = 0, pcbr;
5557         int i;
5558
5559         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5560
5561         gtfifodbg = I915_READ(GTFIFODBG);
5562         if (gtfifodbg) {
5563                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
5564                                  gtfifodbg);
5565                 I915_WRITE(GTFIFODBG, gtfifodbg);
5566         }
5567
5568         cherryview_check_pctx(dev_priv);
5569
5570         /* 1a & 1b: Get forcewake during program sequence. Although the driver
5571          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
5572         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5573
5574         /*  Disable RC states. */
5575         I915_WRITE(GEN6_RC_CONTROL, 0);
5576
5577         /* 2a: Program RC6 thresholds.*/
5578         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
5579         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
5580         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
5581
5582         for_each_ring(ring, dev_priv, i)
5583                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
5584         I915_WRITE(GEN6_RC_SLEEP, 0);
5585
5586         /* TO threshold set to 500 us ( 0x186 * 1.28 us) */
5587         I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
5588
5589         /* allows RC6 residency counter to work */
5590         I915_WRITE(VLV_COUNTER_CONTROL,
5591                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
5592                                       VLV_MEDIA_RC6_COUNT_EN |
5593                                       VLV_RENDER_RC6_COUNT_EN));
5594
5595         /* For now we assume BIOS is allocating and populating the PCBR  */
5596         pcbr = I915_READ(VLV_PCBR);
5597
5598         /* 3: Enable RC6 */
5599         if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
5600                                                 (pcbr >> VLV_PCBR_ADDR_SHIFT))
5601                 rc6_mode = GEN7_RC_CTL_TO_MODE;
5602
5603         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
5604
5605         /* 4 Program defaults and thresholds for RPS*/
5606         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
5607         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
5608         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
5609         I915_WRITE(GEN6_RP_UP_EI, 66000);
5610         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
5611
5612         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
5613
5614         /* 5: Enable RPS */
5615         I915_WRITE(GEN6_RP_CONTROL,
5616                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
5617                    GEN6_RP_MEDIA_IS_GFX |
5618                    GEN6_RP_ENABLE |
5619                    GEN6_RP_UP_BUSY_AVG |
5620                    GEN6_RP_DOWN_IDLE_AVG);
5621
5622         /* Setting Fixed Bias */
5623         val = VLV_OVERRIDE_EN |
5624                   VLV_SOC_TDP_EN |
5625                   CHV_BIAS_CPU_50_SOC_50;
5626         vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
5627
5628         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
5629
5630         /* RPS code assumes GPLL is used */
5631         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
5632
5633         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
5634         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
5635
5636         dev_priv->rps.cur_freq = (val >> 8) & 0xff;
5637         DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
5638                          intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5639                          dev_priv->rps.cur_freq);
5640
5641         DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
5642                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5643                          dev_priv->rps.efficient_freq);
5644
5645         valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
5646
5647         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5648 }
5649
5650 static void valleyview_enable_rps(struct drm_device *dev)
5651 {
5652         struct drm_i915_private *dev_priv = dev->dev_private;
5653         struct intel_engine_cs *ring;
5654         u32 gtfifodbg, val, rc6_mode = 0;
5655         int i;
5656
5657         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5658
5659         valleyview_check_pctx(dev_priv);
5660
5661         if ((gtfifodbg = I915_READ(GTFIFODBG))) {
5662                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
5663                                  gtfifodbg);
5664                 I915_WRITE(GTFIFODBG, gtfifodbg);
5665         }
5666
5667         /* If VLV, Forcewake all wells, else re-direct to regular path */
5668         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5669
5670         /*  Disable RC states. */
5671         I915_WRITE(GEN6_RC_CONTROL, 0);
5672
5673         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
5674         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
5675         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
5676         I915_WRITE(GEN6_RP_UP_EI, 66000);
5677         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
5678
5679         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
5680
5681         I915_WRITE(GEN6_RP_CONTROL,
5682                    GEN6_RP_MEDIA_TURBO |
5683                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
5684                    GEN6_RP_MEDIA_IS_GFX |
5685                    GEN6_RP_ENABLE |
5686                    GEN6_RP_UP_BUSY_AVG |
5687                    GEN6_RP_DOWN_IDLE_CONT);
5688
5689         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
5690         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
5691         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
5692
5693         for_each_ring(ring, dev_priv, i)
5694                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
5695
5696         I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
5697
5698         /* allows RC6 residency counter to work */
5699         I915_WRITE(VLV_COUNTER_CONTROL,
5700                    _MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN |
5701                                       VLV_RENDER_RC0_COUNT_EN |
5702                                       VLV_MEDIA_RC6_COUNT_EN |
5703                                       VLV_RENDER_RC6_COUNT_EN));
5704
5705         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
5706                 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
5707
5708         intel_print_rc6_info(dev, rc6_mode);
5709
5710         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
5711
5712         /* Setting Fixed Bias */
5713         val = VLV_OVERRIDE_EN |
5714                   VLV_SOC_TDP_EN |
5715                   VLV_BIAS_CPU_125_SOC_875;
5716         vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
5717
5718         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
5719
5720         /* RPS code assumes GPLL is used */
5721         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
5722
5723         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
5724         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
5725
5726         dev_priv->rps.cur_freq = (val >> 8) & 0xff;
5727         DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
5728                          intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5729                          dev_priv->rps.cur_freq);
5730
5731         DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
5732                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5733                          dev_priv->rps.efficient_freq);
5734
5735         valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
5736
5737         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5738 }
5739
5740 static unsigned long intel_pxfreq(u32 vidfreq)
5741 {
5742         unsigned long freq;
5743         int div = (vidfreq & 0x3f0000) >> 16;
5744         int post = (vidfreq & 0x3000) >> 12;
5745         int pre = (vidfreq & 0x7);
5746
5747         if (!pre)
5748                 return 0;
5749
5750         freq = ((div * 133333) / ((1<<post) * pre));
5751
5752         return freq;
5753 }
5754
5755 static const struct cparams {
5756         u16 i;
5757         u16 t;
5758         u16 m;
5759         u16 c;
5760 } cparams[] = {
5761         { 1, 1333, 301, 28664 },
5762         { 1, 1066, 294, 24460 },
5763         { 1, 800, 294, 25192 },
5764         { 0, 1333, 276, 27605 },
5765         { 0, 1066, 276, 27605 },
5766         { 0, 800, 231, 23784 },
5767 };
5768
5769 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
5770 {
5771         u64 total_count, diff, ret;
5772         u32 count1, count2, count3, m = 0, c = 0;
5773         unsigned long now = jiffies_to_msecs(jiffies), diff1;
5774         int i;
5775
5776         assert_spin_locked(&mchdev_lock);
5777
5778         diff1 = now - dev_priv->ips.last_time1;
5779
5780         /* Prevent division-by-zero if we are asking too fast.
5781          * Also, we don't get interesting results if we are polling
5782          * faster than once in 10ms, so just return the saved value
5783          * in such cases.
5784          */
5785         if (diff1 <= 10)
5786                 return dev_priv->ips.chipset_power;
5787
5788         count1 = I915_READ(DMIEC);
5789         count2 = I915_READ(DDREC);
5790         count3 = I915_READ(CSIEC);
5791
5792         total_count = count1 + count2 + count3;
5793
5794         /* FIXME: handle per-counter overflow */
5795         if (total_count < dev_priv->ips.last_count1) {
5796                 diff = ~0UL - dev_priv->ips.last_count1;
5797                 diff += total_count;
5798         } else {
5799                 diff = total_count - dev_priv->ips.last_count1;
5800         }
5801
5802         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
5803                 if (cparams[i].i == dev_priv->ips.c_m &&
5804                     cparams[i].t == dev_priv->ips.r_t) {
5805                         m = cparams[i].m;
5806                         c = cparams[i].c;
5807                         break;
5808                 }
5809         }
5810
5811         diff = div_u64(diff, diff1);
5812         ret = ((m * diff) + c);
5813         ret = div_u64(ret, 10);
5814
5815         dev_priv->ips.last_count1 = total_count;
5816         dev_priv->ips.last_time1 = now;
5817
5818         dev_priv->ips.chipset_power = ret;
5819
5820         return ret;
5821 }
5822
5823 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
5824 {
5825         struct drm_device *dev = dev_priv->dev;
5826         unsigned long val;
5827
5828         if (INTEL_INFO(dev)->gen != 5)
5829                 return 0;
5830
5831         spin_lock_irq(&mchdev_lock);
5832
5833         val = __i915_chipset_val(dev_priv);
5834
5835         spin_unlock_irq(&mchdev_lock);
5836
5837         return val;
5838 }
5839
5840 unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
5841 {
5842         unsigned long m, x, b;
5843         u32 tsfs;
5844
5845         tsfs = I915_READ(TSFS);
5846
5847         m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
5848         x = I915_READ8(TR1);
5849
5850         b = tsfs & TSFS_INTR_MASK;
5851
5852         return ((m * x) / 127) - b;
5853 }
5854
5855 static int _pxvid_to_vd(u8 pxvid)
5856 {
5857         if (pxvid == 0)
5858                 return 0;
5859
5860         if (pxvid >= 8 && pxvid < 31)
5861                 pxvid = 31;
5862
5863         return (pxvid + 2) * 125;
5864 }
5865
5866 static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
5867 {
5868         struct drm_device *dev = dev_priv->dev;
5869         const int vd = _pxvid_to_vd(pxvid);
5870         const int vm = vd - 1125;
5871
5872         if (INTEL_INFO(dev)->is_mobile)
5873                 return vm > 0 ? vm : 0;
5874
5875         return vd;
5876 }
5877
5878 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
5879 {
5880         u64 now, diff, diffms;
5881         u32 count;
5882
5883         assert_spin_locked(&mchdev_lock);
5884
5885         now = ktime_get_raw_ns();
5886         diffms = now - dev_priv->ips.last_time2;
5887         do_div(diffms, NSEC_PER_MSEC);
5888
5889         /* Don't divide by 0 */
5890         if (!diffms)
5891                 return;
5892
5893         count = I915_READ(GFXEC);
5894
5895         if (count < dev_priv->ips.last_count2) {
5896                 diff = ~0UL - dev_priv->ips.last_count2;
5897                 diff += count;
5898         } else {
5899                 diff = count - dev_priv->ips.last_count2;
5900         }
5901
5902         dev_priv->ips.last_count2 = count;
5903         dev_priv->ips.last_time2 = now;
5904
5905         /* More magic constants... */
5906         diff = diff * 1181;
5907         diff = div_u64(diff, diffms * 10);
5908         dev_priv->ips.gfx_power = diff;
5909 }
5910
5911 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
5912 {
5913         struct drm_device *dev = dev_priv->dev;
5914
5915         if (INTEL_INFO(dev)->gen != 5)
5916                 return;
5917
5918         spin_lock_irq(&mchdev_lock);
5919
5920         __i915_update_gfx_val(dev_priv);
5921
5922         spin_unlock_irq(&mchdev_lock);
5923 }
5924
5925 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
5926 {
5927         unsigned long t, corr, state1, corr2, state2;
5928         u32 pxvid, ext_v;
5929
5930         assert_spin_locked(&mchdev_lock);
5931
5932         pxvid = I915_READ(PXVFREQ(dev_priv->rps.cur_freq));
5933         pxvid = (pxvid >> 24) & 0x7f;
5934         ext_v = pvid_to_extvid(dev_priv, pxvid);
5935
5936         state1 = ext_v;
5937
5938         t = i915_mch_val(dev_priv);
5939
5940         /* Revel in the empirically derived constants */
5941
5942         /* Correction factor in 1/100000 units */
5943         if (t > 80)
5944                 corr = ((t * 2349) + 135940);
5945         else if (t >= 50)
5946                 corr = ((t * 964) + 29317);
5947         else /* < 50 */
5948                 corr = ((t * 301) + 1004);
5949
5950         corr = corr * ((150142 * state1) / 10000 - 78642);
5951         corr /= 100000;
5952         corr2 = (corr * dev_priv->ips.corr);
5953
5954         state2 = (corr2 * state1) / 10000;
5955         state2 /= 100; /* convert to mW */
5956
5957         __i915_update_gfx_val(dev_priv);
5958
5959         return dev_priv->ips.gfx_power + state2;
5960 }
5961
5962 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
5963 {
5964         struct drm_device *dev = dev_priv->dev;
5965         unsigned long val;
5966
5967         if (INTEL_INFO(dev)->gen != 5)
5968                 return 0;
5969
5970         spin_lock_irq(&mchdev_lock);
5971
5972         val = __i915_gfx_val(dev_priv);
5973
5974         spin_unlock_irq(&mchdev_lock);
5975
5976         return val;
5977 }
5978
5979 /**
5980  * i915_read_mch_val - return value for IPS use
5981  *
5982  * Calculate and return a value for the IPS driver to use when deciding whether
5983  * we have thermal and power headroom to increase CPU or GPU power budget.
5984  */
5985 unsigned long i915_read_mch_val(void)
5986 {
5987         struct drm_i915_private *dev_priv;
5988         unsigned long chipset_val, graphics_val, ret = 0;
5989
5990         spin_lock_irq(&mchdev_lock);
5991         if (!i915_mch_dev)
5992                 goto out_unlock;
5993         dev_priv = i915_mch_dev;
5994
5995         chipset_val = __i915_chipset_val(dev_priv);
5996         graphics_val = __i915_gfx_val(dev_priv);
5997
5998         ret = chipset_val + graphics_val;
5999
6000 out_unlock:
6001         spin_unlock_irq(&mchdev_lock);
6002
6003         return ret;
6004 }
6005 EXPORT_SYMBOL_GPL(i915_read_mch_val);
6006
6007 /**
6008  * i915_gpu_raise - raise GPU frequency limit
6009  *
6010  * Raise the limit; IPS indicates we have thermal headroom.
6011  */
6012 bool i915_gpu_raise(void)
6013 {
6014         struct drm_i915_private *dev_priv;
6015         bool ret = true;
6016
6017         spin_lock_irq(&mchdev_lock);
6018         if (!i915_mch_dev) {
6019                 ret = false;
6020                 goto out_unlock;
6021         }
6022         dev_priv = i915_mch_dev;
6023
6024         if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
6025                 dev_priv->ips.max_delay--;
6026
6027 out_unlock:
6028         spin_unlock_irq(&mchdev_lock);
6029
6030         return ret;
6031 }
6032 EXPORT_SYMBOL_GPL(i915_gpu_raise);
6033
6034 /**
6035  * i915_gpu_lower - lower GPU frequency limit
6036  *
6037  * IPS indicates we're close to a thermal limit, so throttle back the GPU
6038  * frequency maximum.
6039  */
6040 bool i915_gpu_lower(void)
6041 {
6042         struct drm_i915_private *dev_priv;
6043         bool ret = true;
6044
6045         spin_lock_irq(&mchdev_lock);
6046         if (!i915_mch_dev) {
6047                 ret = false;
6048                 goto out_unlock;
6049         }
6050         dev_priv = i915_mch_dev;
6051
6052         if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
6053                 dev_priv->ips.max_delay++;
6054
6055 out_unlock:
6056         spin_unlock_irq(&mchdev_lock);
6057
6058         return ret;
6059 }
6060 EXPORT_SYMBOL_GPL(i915_gpu_lower);
6061
6062 /**
6063  * i915_gpu_busy - indicate GPU business to IPS
6064  *
6065  * Tell the IPS driver whether or not the GPU is busy.
6066  */
6067 bool i915_gpu_busy(void)
6068 {
6069         struct drm_i915_private *dev_priv;
6070         struct intel_engine_cs *ring;
6071         bool ret = false;
6072         int i;
6073
6074         spin_lock_irq(&mchdev_lock);
6075         if (!i915_mch_dev)
6076                 goto out_unlock;
6077         dev_priv = i915_mch_dev;
6078
6079         for_each_ring(ring, dev_priv, i)
6080                 ret |= !list_empty(&ring->request_list);
6081
6082 out_unlock:
6083         spin_unlock_irq(&mchdev_lock);
6084
6085         return ret;
6086 }
6087 EXPORT_SYMBOL_GPL(i915_gpu_busy);
6088
6089 /**
6090  * i915_gpu_turbo_disable - disable graphics turbo
6091  *
6092  * Disable graphics turbo by resetting the max frequency and setting the
6093  * current frequency to the default.
6094  */
6095 bool i915_gpu_turbo_disable(void)
6096 {
6097         struct drm_i915_private *dev_priv;
6098         bool ret = true;
6099
6100         spin_lock_irq(&mchdev_lock);
6101         if (!i915_mch_dev) {
6102                 ret = false;
6103                 goto out_unlock;
6104         }
6105         dev_priv = i915_mch_dev;
6106
6107         dev_priv->ips.max_delay = dev_priv->ips.fstart;
6108
6109         if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
6110                 ret = false;
6111
6112 out_unlock:
6113         spin_unlock_irq(&mchdev_lock);
6114
6115         return ret;
6116 }
6117 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
6118
6119 /**
6120  * Tells the intel_ips driver that the i915 driver is now loaded, if
6121  * IPS got loaded first.
6122  *
6123  * This awkward dance is so that neither module has to depend on the
6124  * other in order for IPS to do the appropriate communication of
6125  * GPU turbo limits to i915.
6126  */
6127 static void
6128 ips_ping_for_i915_load(void)
6129 {
6130         void (*link)(void);
6131
6132         link = symbol_get(ips_link_to_i915_driver);
6133         if (link) {
6134                 link();
6135                 symbol_put(ips_link_to_i915_driver);
6136         }
6137 }
6138
6139 void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
6140 {
6141         /* We only register the i915 ips part with intel-ips once everything is
6142          * set up, to avoid intel-ips sneaking in and reading bogus values. */
6143         spin_lock_irq(&mchdev_lock);
6144         i915_mch_dev = dev_priv;
6145         spin_unlock_irq(&mchdev_lock);
6146
6147         ips_ping_for_i915_load();
6148 }
6149
6150 void intel_gpu_ips_teardown(void)
6151 {
6152         spin_lock_irq(&mchdev_lock);
6153         i915_mch_dev = NULL;
6154         spin_unlock_irq(&mchdev_lock);
6155 }
6156
6157 static void intel_init_emon(struct drm_device *dev)
6158 {
6159         struct drm_i915_private *dev_priv = dev->dev_private;
6160         u32 lcfuse;
6161         u8 pxw[16];
6162         int i;
6163
6164         /* Disable to program */
6165         I915_WRITE(ECR, 0);
6166         POSTING_READ(ECR);
6167
6168         /* Program energy weights for various events */
6169         I915_WRITE(SDEW, 0x15040d00);
6170         I915_WRITE(CSIEW0, 0x007f0000);
6171         I915_WRITE(CSIEW1, 0x1e220004);
6172         I915_WRITE(CSIEW2, 0x04000004);
6173
6174         for (i = 0; i < 5; i++)
6175                 I915_WRITE(PEW(i), 0);
6176         for (i = 0; i < 3; i++)
6177                 I915_WRITE(DEW(i), 0);
6178
6179         /* Program P-state weights to account for frequency power adjustment */
6180         for (i = 0; i < 16; i++) {
6181                 u32 pxvidfreq = I915_READ(PXVFREQ(i));
6182                 unsigned long freq = intel_pxfreq(pxvidfreq);
6183                 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
6184                         PXVFREQ_PX_SHIFT;
6185                 unsigned long val;
6186
6187                 val = vid * vid;
6188                 val *= (freq / 1000);
6189                 val *= 255;
6190                 val /= (127*127*900);
6191                 if (val > 0xff)
6192                         DRM_ERROR("bad pxval: %ld\n", val);
6193                 pxw[i] = val;
6194         }
6195         /* Render standby states get 0 weight */
6196         pxw[14] = 0;
6197         pxw[15] = 0;
6198
6199         for (i = 0; i < 4; i++) {
6200                 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
6201                         (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
6202                 I915_WRITE(PXW(i), val);
6203         }
6204
6205         /* Adjust magic regs to magic values (more experimental results) */
6206         I915_WRITE(OGW0, 0);
6207         I915_WRITE(OGW1, 0);
6208         I915_WRITE(EG0, 0x00007f00);
6209         I915_WRITE(EG1, 0x0000000e);
6210         I915_WRITE(EG2, 0x000e0000);
6211         I915_WRITE(EG3, 0x68000300);
6212         I915_WRITE(EG4, 0x42000000);
6213         I915_WRITE(EG5, 0x00140031);
6214         I915_WRITE(EG6, 0);
6215         I915_WRITE(EG7, 0);
6216
6217         for (i = 0; i < 8; i++)
6218                 I915_WRITE(PXWL(i), 0);
6219
6220         /* Enable PMON + select events */
6221         I915_WRITE(ECR, 0x80000019);
6222
6223         lcfuse = I915_READ(LCFUSE02);
6224
6225         dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
6226 }
6227
6228 void intel_init_gt_powersave(struct drm_device *dev)
6229 {
6230         i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
6231
6232         if (IS_CHERRYVIEW(dev))
6233                 cherryview_init_gt_powersave(dev);
6234         else if (IS_VALLEYVIEW(dev))
6235                 valleyview_init_gt_powersave(dev);
6236 }
6237
6238 void intel_cleanup_gt_powersave(struct drm_device *dev)
6239 {
6240         if (IS_CHERRYVIEW(dev))
6241                 return;
6242         else if (IS_VALLEYVIEW(dev))
6243                 valleyview_cleanup_gt_powersave(dev);
6244 }
6245
6246 static void gen6_suspend_rps(struct drm_device *dev)
6247 {
6248         struct drm_i915_private *dev_priv = dev->dev_private;
6249
6250         flush_delayed_work(&dev_priv->rps.delayed_resume_work);
6251
6252         gen6_disable_rps_interrupts(dev);
6253 }
6254
6255 /**
6256  * intel_suspend_gt_powersave - suspend PM work and helper threads
6257  * @dev: drm device
6258  *
6259  * We don't want to disable RC6 or other features here, we just want
6260  * to make sure any work we've queued has finished and won't bother
6261  * us while we're suspended.
6262  */
6263 void intel_suspend_gt_powersave(struct drm_device *dev)
6264 {
6265         struct drm_i915_private *dev_priv = dev->dev_private;
6266
6267         if (INTEL_INFO(dev)->gen < 6)
6268                 return;
6269
6270         gen6_suspend_rps(dev);
6271
6272         /* Force GPU to min freq during suspend */
6273         gen6_rps_idle(dev_priv);
6274 }
6275
6276 void intel_disable_gt_powersave(struct drm_device *dev)
6277 {
6278         struct drm_i915_private *dev_priv = dev->dev_private;
6279
6280         if (IS_IRONLAKE_M(dev)) {
6281                 ironlake_disable_drps(dev);
6282         } else if (INTEL_INFO(dev)->gen >= 6) {
6283                 intel_suspend_gt_powersave(dev);
6284
6285                 mutex_lock(&dev_priv->rps.hw_lock);
6286                 if (INTEL_INFO(dev)->gen >= 9)
6287                         gen9_disable_rps(dev);
6288                 else if (IS_CHERRYVIEW(dev))
6289                         cherryview_disable_rps(dev);
6290                 else if (IS_VALLEYVIEW(dev))
6291                         valleyview_disable_rps(dev);
6292                 else
6293                         gen6_disable_rps(dev);
6294
6295                 dev_priv->rps.enabled = false;
6296                 mutex_unlock(&dev_priv->rps.hw_lock);
6297         }
6298 }
6299
6300 static void intel_gen6_powersave_work(struct work_struct *work)
6301 {
6302         struct drm_i915_private *dev_priv =
6303                 container_of(work, struct drm_i915_private,
6304                              rps.delayed_resume_work.work);
6305         struct drm_device *dev = dev_priv->dev;
6306
6307         mutex_lock(&dev_priv->rps.hw_lock);
6308
6309         gen6_reset_rps_interrupts(dev);
6310
6311         if (IS_CHERRYVIEW(dev)) {
6312                 cherryview_enable_rps(dev);
6313         } else if (IS_VALLEYVIEW(dev)) {
6314                 valleyview_enable_rps(dev);
6315         } else if (INTEL_INFO(dev)->gen >= 9) {
6316                 gen9_enable_rc6(dev);
6317                 gen9_enable_rps(dev);
6318                 if (IS_SKYLAKE(dev))
6319                         __gen6_update_ring_freq(dev);
6320         } else if (IS_BROADWELL(dev)) {
6321                 gen8_enable_rps(dev);
6322                 __gen6_update_ring_freq(dev);
6323         } else {
6324                 gen6_enable_rps(dev);
6325                 __gen6_update_ring_freq(dev);
6326         }
6327
6328         WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq);
6329         WARN_ON(dev_priv->rps.idle_freq > dev_priv->rps.max_freq);
6330
6331         WARN_ON(dev_priv->rps.efficient_freq < dev_priv->rps.min_freq);
6332         WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq);
6333
6334         dev_priv->rps.enabled = true;
6335
6336         gen6_enable_rps_interrupts(dev);
6337
6338         mutex_unlock(&dev_priv->rps.hw_lock);
6339
6340         intel_runtime_pm_put(dev_priv);
6341 }
6342
6343 void intel_enable_gt_powersave(struct drm_device *dev)
6344 {
6345         struct drm_i915_private *dev_priv = dev->dev_private;
6346
6347         /* Powersaving is controlled by the host when inside a VM */
6348         if (intel_vgpu_active(dev))
6349                 return;
6350
6351         if (IS_IRONLAKE_M(dev)) {
6352                 mutex_lock(&dev->struct_mutex);
6353                 ironlake_enable_drps(dev);
6354                 intel_init_emon(dev);
6355                 mutex_unlock(&dev->struct_mutex);
6356         } else if (INTEL_INFO(dev)->gen >= 6) {
6357                 /*
6358                  * PCU communication is slow and this doesn't need to be
6359                  * done at any specific time, so do this out of our fast path
6360                  * to make resume and init faster.
6361                  *
6362                  * We depend on the HW RC6 power context save/restore
6363                  * mechanism when entering D3 through runtime PM suspend. So
6364                  * disable RPM until RPS/RC6 is properly setup. We can only
6365                  * get here via the driver load/system resume/runtime resume
6366                  * paths, so the _noresume version is enough (and in case of
6367                  * runtime resume it's necessary).
6368                  */
6369                 if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
6370                                            round_jiffies_up_relative(HZ)))
6371                         intel_runtime_pm_get_noresume(dev_priv);
6372         }
6373 }
6374
6375 void intel_reset_gt_powersave(struct drm_device *dev)
6376 {
6377         struct drm_i915_private *dev_priv = dev->dev_private;
6378
6379         if (INTEL_INFO(dev)->gen < 6)
6380                 return;
6381
6382         gen6_suspend_rps(dev);
6383         dev_priv->rps.enabled = false;
6384 }
6385
6386 static void ibx_init_clock_gating(struct drm_device *dev)
6387 {
6388         struct drm_i915_private *dev_priv = dev->dev_private;
6389
6390         /*
6391          * On Ibex Peak and Cougar Point, we need to disable clock
6392          * gating for the panel power sequencer or it will fail to
6393          * start up when no ports are active.
6394          */
6395         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6396 }
6397
6398 static void g4x_disable_trickle_feed(struct drm_device *dev)
6399 {
6400         struct drm_i915_private *dev_priv = dev->dev_private;
6401         enum pipe pipe;
6402
6403         for_each_pipe(dev_priv, pipe) {
6404                 I915_WRITE(DSPCNTR(pipe),
6405                            I915_READ(DSPCNTR(pipe)) |
6406                            DISPPLANE_TRICKLE_FEED_DISABLE);
6407
6408                 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
6409                 POSTING_READ(DSPSURF(pipe));
6410         }
6411 }
6412
6413 static void ilk_init_lp_watermarks(struct drm_device *dev)
6414 {
6415         struct drm_i915_private *dev_priv = dev->dev_private;
6416
6417         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6418         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6419         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6420
6421         /*
6422          * Don't touch WM1S_LP_EN here.
6423          * Doing so could cause underruns.
6424          */
6425 }
6426
6427 static void ironlake_init_clock_gating(struct drm_device *dev)
6428 {
6429         struct drm_i915_private *dev_priv = dev->dev_private;
6430         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6431
6432         /*
6433          * Required for FBC
6434          * WaFbcDisableDpfcClockGating:ilk
6435          */
6436         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6437                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6438                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6439
6440         I915_WRITE(PCH_3DCGDIS0,
6441                    MARIUNIT_CLOCK_GATE_DISABLE |
6442                    SVSMUNIT_CLOCK_GATE_DISABLE);
6443         I915_WRITE(PCH_3DCGDIS1,
6444                    VFMUNIT_CLOCK_GATE_DISABLE);
6445
6446         /*
6447          * According to the spec the following bits should be set in
6448          * order to enable memory self-refresh
6449          * The bit 22/21 of 0x42004
6450          * The bit 5 of 0x42020
6451          * The bit 15 of 0x45000
6452          */
6453         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6454                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
6455                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6456         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6457         I915_WRITE(DISP_ARB_CTL,
6458                    (I915_READ(DISP_ARB_CTL) |
6459                     DISP_FBC_WM_DIS));
6460
6461         ilk_init_lp_watermarks(dev);
6462
6463         /*
6464          * Based on the document from hardware guys the following bits
6465          * should be set unconditionally in order to enable FBC.
6466          * The bit 22 of 0x42000
6467          * The bit 22 of 0x42004
6468          * The bit 7,8,9 of 0x42020.
6469          */
6470         if (IS_IRONLAKE_M(dev)) {
6471                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6472                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6473                            I915_READ(ILK_DISPLAY_CHICKEN1) |
6474                            ILK_FBCQ_DIS);
6475                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6476                            I915_READ(ILK_DISPLAY_CHICKEN2) |
6477                            ILK_DPARB_GATE);
6478         }
6479
6480         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6481
6482         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6483                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6484                    ILK_ELPIN_409_SELECT);
6485         I915_WRITE(_3D_CHICKEN2,
6486                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
6487                    _3D_CHICKEN2_WM_READ_PIPELINED);
6488
6489         /* WaDisableRenderCachePipelinedFlush:ilk */
6490         I915_WRITE(CACHE_MODE_0,
6491                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
6492
6493         /* WaDisable_RenderCache_OperationalFlush:ilk */
6494         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6495
6496         g4x_disable_trickle_feed(dev);
6497
6498         ibx_init_clock_gating(dev);
6499 }
6500
6501 static void cpt_init_clock_gating(struct drm_device *dev)
6502 {
6503         struct drm_i915_private *dev_priv = dev->dev_private;
6504         int pipe;
6505         uint32_t val;
6506
6507         /*
6508          * On Ibex Peak and Cougar Point, we need to disable clock
6509          * gating for the panel power sequencer or it will fail to
6510          * start up when no ports are active.
6511          */
6512         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6513                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6514                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
6515         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
6516                    DPLS_EDP_PPS_FIX_DIS);
6517         /* The below fixes the weird display corruption, a few pixels shifted
6518          * downward, on (only) LVDS of some HP laptops with IVY.
6519          */
6520         for_each_pipe(dev_priv, pipe) {
6521                 val = I915_READ(TRANS_CHICKEN2(pipe));
6522                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6523                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6524                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6525                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6526                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
6527                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6528                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6529                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
6530         }
6531         /* WADP0ClockGatingDisable */
6532         for_each_pipe(dev_priv, pipe) {
6533                 I915_WRITE(TRANS_CHICKEN1(pipe),
6534                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6535         }
6536 }
6537
6538 static void gen6_check_mch_setup(struct drm_device *dev)
6539 {
6540         struct drm_i915_private *dev_priv = dev->dev_private;
6541         uint32_t tmp;
6542
6543         tmp = I915_READ(MCH_SSKPD);
6544         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6545                 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6546                               tmp);
6547 }
6548
6549 static void gen6_init_clock_gating(struct drm_device *dev)
6550 {
6551         struct drm_i915_private *dev_priv = dev->dev_private;
6552         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6553
6554         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6555
6556         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6557                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6558                    ILK_ELPIN_409_SELECT);
6559
6560         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
6561         I915_WRITE(_3D_CHICKEN,
6562                    _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
6563
6564         /* WaDisable_RenderCache_OperationalFlush:snb */
6565         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6566
6567         /*
6568          * BSpec recoomends 8x4 when MSAA is used,
6569          * however in practice 16x4 seems fastest.
6570          *
6571          * Note that PS/WM thread counts depend on the WIZ hashing
6572          * disable bit, which we don't touch here, but it's good
6573          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6574          */
6575         I915_WRITE(GEN6_GT_MODE,
6576                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6577
6578         ilk_init_lp_watermarks(dev);
6579
6580         I915_WRITE(CACHE_MODE_0,
6581                    _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
6582
6583         I915_WRITE(GEN6_UCGCTL1,
6584                    I915_READ(GEN6_UCGCTL1) |
6585                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
6586                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
6587
6588         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
6589          * gating disable must be set.  Failure to set it results in
6590          * flickering pixels due to Z write ordering failures after
6591          * some amount of runtime in the Mesa "fire" demo, and Unigine
6592          * Sanctuary and Tropics, and apparently anything else with
6593          * alpha test or pixel discard.
6594          *
6595          * According to the spec, bit 11 (RCCUNIT) must also be set,
6596          * but we didn't debug actual testcases to find it out.
6597          *
6598          * WaDisableRCCUnitClockGating:snb
6599          * WaDisableRCPBUnitClockGating:snb
6600          */
6601         I915_WRITE(GEN6_UCGCTL2,
6602                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
6603                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
6604
6605         /* WaStripsFansDisableFastClipPerformanceFix:snb */
6606         I915_WRITE(_3D_CHICKEN3,
6607                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
6608
6609         /*
6610          * Bspec says:
6611          * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
6612          * 3DSTATE_SF number of SF output attributes is more than 16."
6613          */
6614         I915_WRITE(_3D_CHICKEN3,
6615                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
6616
6617         /*
6618          * According to the spec the following bits should be
6619          * set in order to enable memory self-refresh and fbc:
6620          * The bit21 and bit22 of 0x42000
6621          * The bit21 and bit22 of 0x42004
6622          * The bit5 and bit7 of 0x42020
6623          * The bit14 of 0x70180
6624          * The bit14 of 0x71180
6625          *
6626          * WaFbcAsynchFlipDisableFbcQueue:snb
6627          */
6628         I915_WRITE(ILK_DISPLAY_CHICKEN1,
6629                    I915_READ(ILK_DISPLAY_CHICKEN1) |
6630                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
6631         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6632                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6633                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
6634         I915_WRITE(ILK_DSPCLK_GATE_D,
6635                    I915_READ(ILK_DSPCLK_GATE_D) |
6636                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
6637                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
6638
6639         g4x_disable_trickle_feed(dev);
6640
6641         cpt_init_clock_gating(dev);
6642
6643         gen6_check_mch_setup(dev);
6644 }
6645
6646 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
6647 {
6648         uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
6649
6650         /*
6651          * WaVSThreadDispatchOverride:ivb,vlv
6652          *
6653          * This actually overrides the dispatch
6654          * mode for all thread types.
6655          */
6656         reg &= ~GEN7_FF_SCHED_MASK;
6657         reg |= GEN7_FF_TS_SCHED_HW;
6658         reg |= GEN7_FF_VS_SCHED_HW;
6659         reg |= GEN7_FF_DS_SCHED_HW;
6660
6661         I915_WRITE(GEN7_FF_THREAD_MODE, reg);
6662 }
6663
6664 static void lpt_init_clock_gating(struct drm_device *dev)
6665 {
6666         struct drm_i915_private *dev_priv = dev->dev_private;
6667
6668         /*
6669          * TODO: this bit should only be enabled when really needed, then
6670          * disabled when not needed anymore in order to save power.
6671          */
6672         if (HAS_PCH_LPT_LP(dev))
6673                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
6674                            I915_READ(SOUTH_DSPCLK_GATE_D) |
6675                            PCH_LP_PARTITION_LEVEL_DISABLE);
6676
6677         /* WADPOClockGatingDisable:hsw */
6678         I915_WRITE(_TRANSA_CHICKEN1,
6679                    I915_READ(_TRANSA_CHICKEN1) |
6680                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6681 }
6682
6683 static void lpt_suspend_hw(struct drm_device *dev)
6684 {
6685         struct drm_i915_private *dev_priv = dev->dev_private;
6686
6687         if (HAS_PCH_LPT_LP(dev)) {
6688                 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
6689
6690                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
6691                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
6692         }
6693 }
6694
6695 static void broadwell_init_clock_gating(struct drm_device *dev)
6696 {
6697         struct drm_i915_private *dev_priv = dev->dev_private;
6698         enum pipe pipe;
6699         uint32_t misccpctl;
6700
6701         ilk_init_lp_watermarks(dev);
6702
6703         /* WaSwitchSolVfFArbitrationPriority:bdw */
6704         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
6705
6706         /* WaPsrDPAMaskVBlankInSRD:bdw */
6707         I915_WRITE(CHICKEN_PAR1_1,
6708                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
6709
6710         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
6711         for_each_pipe(dev_priv, pipe) {
6712                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
6713                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
6714                            BDW_DPRS_MASK_VBLANK_SRD);
6715         }
6716
6717         /* WaVSRefCountFullforceMissDisable:bdw */
6718         /* WaDSRefCountFullforceMissDisable:bdw */
6719         I915_WRITE(GEN7_FF_THREAD_MODE,
6720                    I915_READ(GEN7_FF_THREAD_MODE) &
6721                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
6722
6723         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6724                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
6725
6726         /* WaDisableSDEUnitClockGating:bdw */
6727         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6728                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6729
6730         /*
6731          * WaProgramL3SqcReg1Default:bdw
6732          * WaTempDisableDOPClkGating:bdw
6733          */
6734         misccpctl = I915_READ(GEN7_MISCCPCTL);
6735         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
6736         I915_WRITE(GEN8_L3SQCREG1, BDW_WA_L3SQCREG1_DEFAULT);
6737         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
6738
6739         /*
6740          * WaGttCachingOffByDefault:bdw
6741          * GTT cache may not work with big pages, so if those
6742          * are ever enabled GTT cache may need to be disabled.
6743          */
6744         I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
6745
6746         lpt_init_clock_gating(dev);
6747 }
6748
6749 static void haswell_init_clock_gating(struct drm_device *dev)
6750 {
6751         struct drm_i915_private *dev_priv = dev->dev_private;
6752
6753         ilk_init_lp_watermarks(dev);
6754
6755         /* L3 caching of data atomics doesn't work -- disable it. */
6756         I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
6757         I915_WRITE(HSW_ROW_CHICKEN3,
6758                    _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
6759
6760         /* This is required by WaCatErrorRejectionIssue:hsw */
6761         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6762                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6763                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6764
6765         /* WaVSRefCountFullforceMissDisable:hsw */
6766         I915_WRITE(GEN7_FF_THREAD_MODE,
6767                    I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
6768
6769         /* WaDisable_RenderCache_OperationalFlush:hsw */
6770         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6771
6772         /* enable HiZ Raw Stall Optimization */
6773         I915_WRITE(CACHE_MODE_0_GEN7,
6774                    _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6775
6776         /* WaDisable4x2SubspanOptimization:hsw */
6777         I915_WRITE(CACHE_MODE_1,
6778                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6779
6780         /*
6781          * BSpec recommends 8x4 when MSAA is used,
6782          * however in practice 16x4 seems fastest.
6783          *
6784          * Note that PS/WM thread counts depend on the WIZ hashing
6785          * disable bit, which we don't touch here, but it's good
6786          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6787          */
6788         I915_WRITE(GEN7_GT_MODE,
6789                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6790
6791         /* WaSampleCChickenBitEnable:hsw */
6792         I915_WRITE(HALF_SLICE_CHICKEN3,
6793                    _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
6794
6795         /* WaSwitchSolVfFArbitrationPriority:hsw */
6796         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
6797
6798         /* WaRsPkgCStateDisplayPMReq:hsw */
6799         I915_WRITE(CHICKEN_PAR1_1,
6800                    I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
6801
6802         lpt_init_clock_gating(dev);
6803 }
6804
6805 static void ivybridge_init_clock_gating(struct drm_device *dev)
6806 {
6807         struct drm_i915_private *dev_priv = dev->dev_private;
6808         uint32_t snpcr;
6809
6810         ilk_init_lp_watermarks(dev);
6811
6812         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
6813
6814         /* WaDisableEarlyCull:ivb */
6815         I915_WRITE(_3D_CHICKEN3,
6816                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6817
6818         /* WaDisableBackToBackFlipFix:ivb */
6819         I915_WRITE(IVB_CHICKEN3,
6820                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6821                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
6822
6823         /* WaDisablePSDDualDispatchEnable:ivb */
6824         if (IS_IVB_GT1(dev))
6825                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
6826                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
6827
6828         /* WaDisable_RenderCache_OperationalFlush:ivb */
6829         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6830
6831         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
6832         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
6833                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
6834
6835         /* WaApplyL3ControlAndL3ChickenMode:ivb */
6836         I915_WRITE(GEN7_L3CNTLREG1,
6837                         GEN7_WA_FOR_GEN7_L3_CONTROL);
6838         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
6839                    GEN7_WA_L3_CHICKEN_MODE);
6840         if (IS_IVB_GT1(dev))
6841                 I915_WRITE(GEN7_ROW_CHICKEN2,
6842                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6843         else {
6844                 /* must write both registers */
6845                 I915_WRITE(GEN7_ROW_CHICKEN2,
6846                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6847                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
6848                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6849         }
6850
6851         /* WaForceL3Serialization:ivb */
6852         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6853                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6854
6855         /*
6856          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
6857          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
6858          */
6859         I915_WRITE(GEN6_UCGCTL2,
6860                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
6861
6862         /* This is required by WaCatErrorRejectionIssue:ivb */
6863         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6864                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6865                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6866
6867         g4x_disable_trickle_feed(dev);
6868
6869         gen7_setup_fixed_func_scheduler(dev_priv);
6870
6871         if (0) { /* causes HiZ corruption on ivb:gt1 */
6872                 /* enable HiZ Raw Stall Optimization */
6873                 I915_WRITE(CACHE_MODE_0_GEN7,
6874                            _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6875         }
6876
6877         /* WaDisable4x2SubspanOptimization:ivb */
6878         I915_WRITE(CACHE_MODE_1,
6879                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6880
6881         /*
6882          * BSpec recommends 8x4 when MSAA is used,
6883          * however in practice 16x4 seems fastest.
6884          *
6885          * Note that PS/WM thread counts depend on the WIZ hashing
6886          * disable bit, which we don't touch here, but it's good
6887          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6888          */
6889         I915_WRITE(GEN7_GT_MODE,
6890                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6891
6892         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
6893         snpcr &= ~GEN6_MBC_SNPCR_MASK;
6894         snpcr |= GEN6_MBC_SNPCR_MED;
6895         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
6896
6897         if (!HAS_PCH_NOP(dev))
6898                 cpt_init_clock_gating(dev);
6899
6900         gen6_check_mch_setup(dev);
6901 }
6902
6903 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
6904 {
6905         I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
6906
6907         /*
6908          * Disable trickle feed and enable pnd deadline calculation
6909          */
6910         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
6911         I915_WRITE(CBR1_VLV, 0);
6912 }
6913
6914 static void valleyview_init_clock_gating(struct drm_device *dev)
6915 {
6916         struct drm_i915_private *dev_priv = dev->dev_private;
6917
6918         vlv_init_display_clock_gating(dev_priv);
6919
6920         /* WaDisableEarlyCull:vlv */
6921         I915_WRITE(_3D_CHICKEN3,
6922                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6923
6924         /* WaDisableBackToBackFlipFix:vlv */
6925         I915_WRITE(IVB_CHICKEN3,
6926                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6927                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
6928
6929         /* WaPsdDispatchEnable:vlv */
6930         /* WaDisablePSDDualDispatchEnable:vlv */
6931         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
6932                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
6933                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
6934
6935         /* WaDisable_RenderCache_OperationalFlush:vlv */
6936         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6937
6938         /* WaForceL3Serialization:vlv */
6939         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6940                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6941
6942         /* WaDisableDopClockGating:vlv */
6943         I915_WRITE(GEN7_ROW_CHICKEN2,
6944                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6945
6946         /* This is required by WaCatErrorRejectionIssue:vlv */
6947         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6948                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6949                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6950
6951         gen7_setup_fixed_func_scheduler(dev_priv);
6952
6953         /*
6954          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
6955          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
6956          */
6957         I915_WRITE(GEN6_UCGCTL2,
6958                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
6959
6960         /* WaDisableL3Bank2xClockGate:vlv
6961          * Disabling L3 clock gating- MMIO 940c[25] = 1
6962          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
6963         I915_WRITE(GEN7_UCGCTL4,
6964                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
6965
6966         /*
6967          * BSpec says this must be set, even though
6968          * WaDisable4x2SubspanOptimization isn't listed for VLV.
6969          */
6970         I915_WRITE(CACHE_MODE_1,
6971                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6972
6973         /*
6974          * BSpec recommends 8x4 when MSAA is used,
6975          * however in practice 16x4 seems fastest.
6976          *
6977          * Note that PS/WM thread counts depend on the WIZ hashing
6978          * disable bit, which we don't touch here, but it's good
6979          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6980          */
6981         I915_WRITE(GEN7_GT_MODE,
6982                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6983
6984         /*
6985          * WaIncreaseL3CreditsForVLVB0:vlv
6986          * This is the hardware default actually.
6987          */
6988         I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
6989
6990         /*
6991          * WaDisableVLVClockGating_VBIIssue:vlv
6992          * Disable clock gating on th GCFG unit to prevent a delay
6993          * in the reporting of vblank events.
6994          */
6995         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
6996 }
6997
6998 static void cherryview_init_clock_gating(struct drm_device *dev)
6999 {
7000         struct drm_i915_private *dev_priv = dev->dev_private;
7001
7002         vlv_init_display_clock_gating(dev_priv);
7003
7004         /* WaVSRefCountFullforceMissDisable:chv */
7005         /* WaDSRefCountFullforceMissDisable:chv */
7006         I915_WRITE(GEN7_FF_THREAD_MODE,
7007                    I915_READ(GEN7_FF_THREAD_MODE) &
7008                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7009
7010         /* WaDisableSemaphoreAndSyncFlipWait:chv */
7011         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7012                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7013
7014         /* WaDisableCSUnitClockGating:chv */
7015         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7016                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7017
7018         /* WaDisableSDEUnitClockGating:chv */
7019         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7020                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7021
7022         /*
7023          * GTT cache may not work with big pages, so if those
7024          * are ever enabled GTT cache may need to be disabled.
7025          */
7026         I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
7027 }
7028
7029 static void g4x_init_clock_gating(struct drm_device *dev)
7030 {
7031         struct drm_i915_private *dev_priv = dev->dev_private;
7032         uint32_t dspclk_gate;
7033
7034         I915_WRITE(RENCLK_GATE_D1, 0);
7035         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7036                    GS_UNIT_CLOCK_GATE_DISABLE |
7037                    CL_UNIT_CLOCK_GATE_DISABLE);
7038         I915_WRITE(RAMCLK_GATE_D, 0);
7039         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7040                 OVRUNIT_CLOCK_GATE_DISABLE |
7041                 OVCUNIT_CLOCK_GATE_DISABLE;
7042         if (IS_GM45(dev))
7043                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7044         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7045
7046         /* WaDisableRenderCachePipelinedFlush */
7047         I915_WRITE(CACHE_MODE_0,
7048                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
7049
7050         /* WaDisable_RenderCache_OperationalFlush:g4x */
7051         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7052
7053         g4x_disable_trickle_feed(dev);
7054 }
7055
7056 static void crestline_init_clock_gating(struct drm_device *dev)
7057 {
7058         struct drm_i915_private *dev_priv = dev->dev_private;
7059
7060         I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7061         I915_WRITE(RENCLK_GATE_D2, 0);
7062         I915_WRITE(DSPCLK_GATE_D, 0);
7063         I915_WRITE(RAMCLK_GATE_D, 0);
7064         I915_WRITE16(DEUC, 0);
7065         I915_WRITE(MI_ARB_STATE,
7066                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7067
7068         /* WaDisable_RenderCache_OperationalFlush:gen4 */
7069         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7070 }
7071
7072 static void broadwater_init_clock_gating(struct drm_device *dev)
7073 {
7074         struct drm_i915_private *dev_priv = dev->dev_private;
7075
7076         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7077                    I965_RCC_CLOCK_GATE_DISABLE |
7078                    I965_RCPB_CLOCK_GATE_DISABLE |
7079                    I965_ISC_CLOCK_GATE_DISABLE |
7080                    I965_FBC_CLOCK_GATE_DISABLE);
7081         I915_WRITE(RENCLK_GATE_D2, 0);
7082         I915_WRITE(MI_ARB_STATE,
7083                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7084
7085         /* WaDisable_RenderCache_OperationalFlush:gen4 */
7086         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7087 }
7088
7089 static void gen3_init_clock_gating(struct drm_device *dev)
7090 {
7091         struct drm_i915_private *dev_priv = dev->dev_private;
7092         u32 dstate = I915_READ(D_STATE);
7093
7094         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7095                 DSTATE_DOT_CLOCK_GATING;
7096         I915_WRITE(D_STATE, dstate);
7097
7098         if (IS_PINEVIEW(dev))
7099                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7100
7101         /* IIR "flip pending" means done if this bit is set */
7102         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7103
7104         /* interrupts should cause a wake up from C3 */
7105         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7106
7107         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7108         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7109
7110         I915_WRITE(MI_ARB_STATE,
7111                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7112 }
7113
7114 static void i85x_init_clock_gating(struct drm_device *dev)
7115 {
7116         struct drm_i915_private *dev_priv = dev->dev_private;
7117
7118         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7119
7120         /* interrupts should cause a wake up from C3 */
7121         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7122                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7123
7124         I915_WRITE(MEM_MODE,
7125                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7126 }
7127
7128 static void i830_init_clock_gating(struct drm_device *dev)
7129 {
7130         struct drm_i915_private *dev_priv = dev->dev_private;
7131
7132         I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
7133
7134         I915_WRITE(MEM_MODE,
7135                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7136                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7137 }
7138
7139 void intel_init_clock_gating(struct drm_device *dev)
7140 {
7141         struct drm_i915_private *dev_priv = dev->dev_private;
7142
7143         if (dev_priv->display.init_clock_gating)
7144                 dev_priv->display.init_clock_gating(dev);
7145 }
7146
7147 void intel_suspend_hw(struct drm_device *dev)
7148 {
7149         if (HAS_PCH_LPT(dev))
7150                 lpt_suspend_hw(dev);
7151 }
7152
7153 /* Set up chip specific power management-related functions */
7154 void intel_init_pm(struct drm_device *dev)
7155 {
7156         struct drm_i915_private *dev_priv = dev->dev_private;
7157
7158         intel_fbc_init(dev_priv);
7159
7160         /* For cxsr */
7161         if (IS_PINEVIEW(dev))
7162                 i915_pineview_get_mem_freq(dev);
7163         else if (IS_GEN5(dev))
7164                 i915_ironlake_get_mem_freq(dev);
7165
7166         /* For FIFO watermark updates */
7167         if (INTEL_INFO(dev)->gen >= 9) {
7168                 skl_setup_wm_latency(dev);
7169
7170                 if (IS_BROXTON(dev))
7171                         dev_priv->display.init_clock_gating =
7172                                 bxt_init_clock_gating;
7173                 else if (IS_SKYLAKE(dev))
7174                         dev_priv->display.init_clock_gating =
7175                                 skl_init_clock_gating;
7176                 dev_priv->display.update_wm = skl_update_wm;
7177                 dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
7178         } else if (HAS_PCH_SPLIT(dev)) {
7179                 ilk_setup_wm_latency(dev);
7180
7181                 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
7182                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7183                     (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
7184                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7185                         dev_priv->display.update_wm = ilk_update_wm;
7186                         dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
7187                 } else {
7188                         DRM_DEBUG_KMS("Failed to read display plane latency. "
7189                                       "Disable CxSR\n");
7190                 }
7191
7192                 if (IS_GEN5(dev))
7193                         dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
7194                 else if (IS_GEN6(dev))
7195                         dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7196                 else if (IS_IVYBRIDGE(dev))
7197                         dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
7198                 else if (IS_HASWELL(dev))
7199                         dev_priv->display.init_clock_gating = haswell_init_clock_gating;
7200                 else if (INTEL_INFO(dev)->gen == 8)
7201                         dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
7202         } else if (IS_CHERRYVIEW(dev)) {
7203                 vlv_setup_wm_latency(dev);
7204
7205                 dev_priv->display.update_wm = vlv_update_wm;
7206                 dev_priv->display.init_clock_gating =
7207                         cherryview_init_clock_gating;
7208         } else if (IS_VALLEYVIEW(dev)) {
7209                 vlv_setup_wm_latency(dev);
7210
7211                 dev_priv->display.update_wm = vlv_update_wm;
7212                 dev_priv->display.init_clock_gating =
7213                         valleyview_init_clock_gating;
7214         } else if (IS_PINEVIEW(dev)) {
7215                 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
7216                                             dev_priv->is_ddr3,
7217                                             dev_priv->fsb_freq,
7218                                             dev_priv->mem_freq)) {
7219                         DRM_INFO("failed to find known CxSR latency "
7220                                  "(found ddr%s fsb freq %d, mem freq %d), "
7221                                  "disabling CxSR\n",
7222                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
7223                                  dev_priv->fsb_freq, dev_priv->mem_freq);
7224                         /* Disable CxSR and never update its watermark again */
7225                         intel_set_memory_cxsr(dev_priv, false);
7226                         dev_priv->display.update_wm = NULL;
7227                 } else
7228                         dev_priv->display.update_wm = pineview_update_wm;
7229                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7230         } else if (IS_G4X(dev)) {
7231                 dev_priv->display.update_wm = g4x_update_wm;
7232                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7233         } else if (IS_GEN4(dev)) {
7234                 dev_priv->display.update_wm = i965_update_wm;
7235                 if (IS_CRESTLINE(dev))
7236                         dev_priv->display.init_clock_gating = crestline_init_clock_gating;
7237                 else if (IS_BROADWATER(dev))
7238                         dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
7239         } else if (IS_GEN3(dev)) {
7240                 dev_priv->display.update_wm = i9xx_update_wm;
7241                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7242                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7243         } else if (IS_GEN2(dev)) {
7244                 if (INTEL_INFO(dev)->num_pipes == 1) {
7245                         dev_priv->display.update_wm = i845_update_wm;
7246                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
7247                 } else {
7248                         dev_priv->display.update_wm = i9xx_update_wm;
7249                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
7250                 }
7251
7252                 if (IS_I85X(dev) || IS_I865G(dev))
7253                         dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7254                 else
7255                         dev_priv->display.init_clock_gating = i830_init_clock_gating;
7256         } else {
7257                 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
7258         }
7259 }
7260
7261 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
7262 {
7263         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
7264
7265         if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
7266                 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
7267                 return -EAGAIN;
7268         }
7269
7270         I915_WRITE(GEN6_PCODE_DATA, *val);
7271         I915_WRITE(GEN6_PCODE_DATA1, 0);
7272         I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
7273
7274         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
7275                      500)) {
7276                 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
7277                 return -ETIMEDOUT;
7278         }
7279
7280         *val = I915_READ(GEN6_PCODE_DATA);
7281         I915_WRITE(GEN6_PCODE_DATA, 0);
7282
7283         return 0;
7284 }
7285
7286 int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
7287 {
7288         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
7289
7290         if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
7291                 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
7292                 return -EAGAIN;
7293         }
7294
7295         I915_WRITE(GEN6_PCODE_DATA, val);
7296         I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
7297
7298         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
7299                      500)) {
7300                 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
7301                 return -ETIMEDOUT;
7302         }
7303
7304         I915_WRITE(GEN6_PCODE_DATA, 0);
7305
7306         return 0;
7307 }
7308
7309 static int vlv_gpu_freq_div(unsigned int czclk_freq)
7310 {
7311         switch (czclk_freq) {
7312         case 200:
7313                 return 10;
7314         case 267:
7315                 return 12;
7316         case 320:
7317         case 333:
7318                 return 16;
7319         case 400:
7320                 return 20;
7321         default:
7322                 return -1;
7323         }
7324 }
7325
7326 static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
7327 {
7328         int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
7329
7330         div = vlv_gpu_freq_div(czclk_freq);
7331         if (div < 0)
7332                 return div;
7333
7334         return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div);
7335 }
7336
7337 static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
7338 {
7339         int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
7340
7341         mul = vlv_gpu_freq_div(czclk_freq);
7342         if (mul < 0)
7343                 return mul;
7344
7345         return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6;
7346 }
7347
7348 static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
7349 {
7350         int div, czclk_freq = dev_priv->rps.cz_freq;
7351
7352         div = vlv_gpu_freq_div(czclk_freq) / 2;
7353         if (div < 0)
7354                 return div;
7355
7356         return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2;
7357 }
7358
7359 static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
7360 {
7361         int mul, czclk_freq = dev_priv->rps.cz_freq;
7362
7363         mul = vlv_gpu_freq_div(czclk_freq) / 2;
7364         if (mul < 0)
7365                 return mul;
7366
7367         /* CHV needs even values */
7368         return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2;
7369 }
7370
7371 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
7372 {
7373         if (IS_GEN9(dev_priv->dev))
7374                 return (val * GT_FREQUENCY_MULTIPLIER) / GEN9_FREQ_SCALER;
7375         else if (IS_CHERRYVIEW(dev_priv->dev))
7376                 return chv_gpu_freq(dev_priv, val);
7377         else if (IS_VALLEYVIEW(dev_priv->dev))
7378                 return byt_gpu_freq(dev_priv, val);
7379         else
7380                 return val * GT_FREQUENCY_MULTIPLIER;
7381 }
7382
7383 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
7384 {
7385         if (IS_GEN9(dev_priv->dev))
7386                 return (val * GEN9_FREQ_SCALER) / GT_FREQUENCY_MULTIPLIER;
7387         else if (IS_CHERRYVIEW(dev_priv->dev))
7388                 return chv_freq_opcode(dev_priv, val);
7389         else if (IS_VALLEYVIEW(dev_priv->dev))
7390                 return byt_freq_opcode(dev_priv, val);
7391         else
7392                 return val / GT_FREQUENCY_MULTIPLIER;
7393 }
7394
7395 struct request_boost {
7396         struct work_struct work;
7397         struct drm_i915_gem_request *req;
7398 };
7399
7400 static void __intel_rps_boost_work(struct work_struct *work)
7401 {
7402         struct request_boost *boost = container_of(work, struct request_boost, work);
7403         struct drm_i915_gem_request *req = boost->req;
7404
7405         if (!i915_gem_request_completed(req, true))
7406                 gen6_rps_boost(to_i915(req->ring->dev), NULL,
7407                                req->emitted_jiffies);
7408
7409         i915_gem_request_unreference__unlocked(req);
7410         kfree(boost);
7411 }
7412
7413 void intel_queue_rps_boost_for_request(struct drm_device *dev,
7414                                        struct drm_i915_gem_request *req)
7415 {
7416         struct request_boost *boost;
7417
7418         if (req == NULL || INTEL_INFO(dev)->gen < 6)
7419                 return;
7420
7421         if (i915_gem_request_completed(req, true))
7422                 return;
7423
7424         boost = kmalloc(sizeof(*boost), GFP_ATOMIC);
7425         if (boost == NULL)
7426                 return;
7427
7428         i915_gem_request_reference(req);
7429         boost->req = req;
7430
7431         INIT_WORK(&boost->work, __intel_rps_boost_work);
7432         queue_work(to_i915(dev)->wq, &boost->work);
7433 }
7434
7435 void intel_pm_setup(struct drm_device *dev)
7436 {
7437         struct drm_i915_private *dev_priv = dev->dev_private;
7438
7439         mutex_init(&dev_priv->rps.hw_lock);
7440         spin_lock_init(&dev_priv->rps.client_lock);
7441
7442         INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
7443                           intel_gen6_powersave_work);
7444         INIT_LIST_HEAD(&dev_priv->rps.clients);
7445         INIT_LIST_HEAD(&dev_priv->rps.semaphores.link);
7446         INIT_LIST_HEAD(&dev_priv->rps.mmioflips.link);
7447
7448         dev_priv->pm.suspended = false;
7449 }