drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/cpufreq.h>
  29 #include <drm/drm_plane_helper.h>
  30 #include "i915_drv.h"
  31 #include "intel_drv.h"
  32 #include "../../../platform/x86/intel_ips.h"
  33 #include <linux/module.h>
  34 #include <drm/drm_atomic_helper.h>
  35
  36 /**
  37  * DOC: RC6
  38  *
  39  * RC6 is a special power stage which allows the GPU to enter an very
  40  * low-voltage mode when idle, using down to 0V while at this stage.  This
  41  * stage is entered automatically when the GPU is idle when RC6 support is
  42  * enabled, and as soon as new workload arises GPU wakes up automatically as well.
  43  *
  44  * There are different RC6 modes available in Intel GPU, which differentiate
  45  * among each other with the latency required to enter and leave RC6 and
  46  * voltage consumed by the GPU in different states.
  47  *
  48  * The combination of the following flags define which states GPU is allowed
  49  * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
  50  * RC6pp is deepest RC6. Their support by hardware varies according to the
  51  * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
  52  * which brings the most power savings; deeper states save more power, but
  53  * require higher latency to switch to and wake up.
  54  */
  55
  56 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  57 {
  58         if (HAS_LLC(dev_priv)) {
  59                 /*
  60                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  61                  * Display WA #0390: skl,kbl
  62                  *
  63                  * Must match Sampler, Pixel Back End, and Media. See
  64                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  65                  */
  66                 I915_WRITE(CHICKEN_PAR1_1,
  67                            I915_READ(CHICKEN_PAR1_1) |
  68                            SKL_DE_COMPRESSED_HASH_MODE);
  69         }
  70
  71         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  72         I915_WRITE(CHICKEN_PAR1_1,
  73                    I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
  74
  75         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  76         I915_WRITE(GEN8_CHICKEN_DCPR_1,
  77                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
  78
  79         /* WaFbcTurnOffFbcWatermark:skl,bxt,kbl,cfl */
  80         /* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */
  81         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  82                    DISP_FBC_WM_DIS |
  83                    DISP_FBC_MEMORY_WAKE);
  84
  85         /* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */
  86         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
  87                    ILK_DPFC_DISABLE_DUMMY0);
  88
  89         if (IS_SKYLAKE(dev_priv)) {
  90                 /* WaDisableDopClockGating */
  91                 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
  92                            & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  93         }
  94 }
  95
  96 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
  97 {
  98         gen9_init_clock_gating(dev_priv);
  99
 100         /* WaDisableSDEUnitClockGating:bxt */
 101         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 102                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 103
 104         /*
 105          * FIXME:
 106          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 107          */
 108         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 109                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 110
 111         /*
 112          * Wa: Backlight PWM may stop in the asserted state, causing backlight
 113          * to stay fully on.
 114          */
 115         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 116                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 117
 118         /*
 119          * Lower the display internal timeout.
 120          * This is needed to avoid any hard hangs when DSI port PLL
 121          * is off and a MMIO access is attempted by any privilege
 122          * application, using batch buffers or any other means.
 123          */
 124         I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 125 }
 126
 127 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 128 {
 129         gen9_init_clock_gating(dev_priv);
 130
 131         /*
 132          * WaDisablePWMClockGating:glk
 133          * Backlight PWM may stop in the asserted state, causing backlight
 134          * to stay fully on.
 135          */
 136         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 137                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 138
 139         /* WaDDIIOTimeout:glk */
 140         if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) {
 141                 u32 val = I915_READ(CHICKEN_MISC_2);
 142                 val &= ~(GLK_CL0_PWR_DOWN |
 143                          GLK_CL1_PWR_DOWN |
 144                          GLK_CL2_PWR_DOWN);
 145                 I915_WRITE(CHICKEN_MISC_2, val);
 146         }
 147
 148 }
 149
 150 static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv)
 151 {
 152         u32 tmp;
 153
 154         tmp = I915_READ(CLKCFG);
 155
 156         switch (tmp & CLKCFG_FSB_MASK) {
 157         case CLKCFG_FSB_533:
 158                 dev_priv->fsb_freq = 533; /* 133*4 */
 159                 break;
 160         case CLKCFG_FSB_800:
 161                 dev_priv->fsb_freq = 800; /* 200*4 */
 162                 break;
 163         case CLKCFG_FSB_667:
 164                 dev_priv->fsb_freq =  667; /* 167*4 */
 165                 break;
 166         case CLKCFG_FSB_400:
 167                 dev_priv->fsb_freq = 400; /* 100*4 */
 168                 break;
 169         }
 170
 171         switch (tmp & CLKCFG_MEM_MASK) {
 172         case CLKCFG_MEM_533:
 173                 dev_priv->mem_freq = 533;
 174                 break;
 175         case CLKCFG_MEM_667:
 176                 dev_priv->mem_freq = 667;
 177                 break;
 178         case CLKCFG_MEM_800:
 179                 dev_priv->mem_freq = 800;
 180                 break;
 181         }
 182
 183         /* detect pineview DDR3 setting */
 184         tmp = I915_READ(CSHRDDR3CTL);
 185         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 186 }
 187
 188 static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
 189 {
 190         u16 ddrpll, csipll;
 191
 192         ddrpll = I915_READ16(DDRMPLL1);
 193         csipll = I915_READ16(CSIPLL0);
 194
 195         switch (ddrpll & 0xff) {
 196         case 0xc:
 197                 dev_priv->mem_freq = 800;
 198                 break;
 199         case 0x10:
 200                 dev_priv->mem_freq = 1066;
 201                 break;
 202         case 0x14:
 203                 dev_priv->mem_freq = 1333;
 204                 break;
 205         case 0x18:
 206                 dev_priv->mem_freq = 1600;
 207                 break;
 208         default:
 209                 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
 210                                  ddrpll & 0xff);
 211                 dev_priv->mem_freq = 0;
 212                 break;
 213         }
 214
 215         dev_priv->ips.r_t = dev_priv->mem_freq;
 216
 217         switch (csipll & 0x3ff) {
 218         case 0x00c:
 219                 dev_priv->fsb_freq = 3200;
 220                 break;
 221         case 0x00e:
 222                 dev_priv->fsb_freq = 3733;
 223                 break;
 224         case 0x010:
 225                 dev_priv->fsb_freq = 4266;
 226                 break;
 227         case 0x012:
 228                 dev_priv->fsb_freq = 4800;
 229                 break;
 230         case 0x014:
 231                 dev_priv->fsb_freq = 5333;
 232                 break;
 233         case 0x016:
 234                 dev_priv->fsb_freq = 5866;
 235                 break;
 236         case 0x018:
 237                 dev_priv->fsb_freq = 6400;
 238                 break;
 239         default:
 240                 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
 241                                  csipll & 0x3ff);
 242                 dev_priv->fsb_freq = 0;
 243                 break;
 244         }
 245
 246         if (dev_priv->fsb_freq == 3200) {
 247                 dev_priv->ips.c_m = 0;
 248         } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
 249                 dev_priv->ips.c_m = 1;
 250         } else {
 251                 dev_priv->ips.c_m = 2;
 252         }
 253 }
 254
 255 static const struct cxsr_latency cxsr_latency_table[] = {
 256         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 257         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 258         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 259         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 260         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 261
 262         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 263         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 264         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 265         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 266         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 267
 268         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 269         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 270         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 271         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 272         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 273
 274         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 275         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 276         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 277         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 278         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 279
 280         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 281         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 282         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 283         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 284         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 285
 286         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 287         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 288         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 289         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 290         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 291 };
 292
 293 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 294                                                          bool is_ddr3,
 295                                                          int fsb,
 296                                                          int mem)
 297 {
 298         const struct cxsr_latency *latency;
 299         int i;
 300
 301         if (fsb == 0 || mem == 0)
 302                 return NULL;
 303
 304         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 305                 latency = &cxsr_latency_table[i];
 306                 if (is_desktop == latency->is_desktop &&
 307                     is_ddr3 == latency->is_ddr3 &&
 308                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 309                         return latency;
 310         }
 311
 312         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 313
 314         return NULL;
 315 }
 316
 317 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 318 {
 319         u32 val;
 320
 321         mutex_lock(&dev_priv->pcu_lock);
 322
 323         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 324         if (enable)
 325                 val &= ~FORCE_DDR_HIGH_FREQ;
 326         else
 327                 val |= FORCE_DDR_HIGH_FREQ;
 328         val &= ~FORCE_DDR_LOW_FREQ;
 329         val |= FORCE_DDR_FREQ_REQ_ACK;
 330         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 331
 332         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 333                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 334                 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
 335
 336         mutex_unlock(&dev_priv->pcu_lock);
 337 }
 338
 339 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 340 {
 341         u32 val;
 342
 343         mutex_lock(&dev_priv->pcu_lock);
 344
 345         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
 346         if (enable)
 347                 val |= DSP_MAXFIFO_PM5_ENABLE;
 348         else
 349                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 350         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
 351
 352         mutex_unlock(&dev_priv->pcu_lock);
 353 }
 354
 355 #define FW_WM(value, plane) \
 356         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 357
 358 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 359 {
 360         bool was_enabled;
 361         u32 val;
 362
 363         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 364                 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 365                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 366                 POSTING_READ(FW_BLC_SELF_VLV);
 367         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 368                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 369                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 370                 POSTING_READ(FW_BLC_SELF);
 371         } else if (IS_PINEVIEW(dev_priv)) {
 372                 val = I915_READ(DSPFW3);
 373                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 374                 if (enable)
 375                         val |= PINEVIEW_SELF_REFRESH_EN;
 376                 else
 377                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 378                 I915_WRITE(DSPFW3, val);
 379                 POSTING_READ(DSPFW3);
 380         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 381                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 382                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 383                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 384                 I915_WRITE(FW_BLC_SELF, val);
 385                 POSTING_READ(FW_BLC_SELF);
 386         } else if (IS_I915GM(dev_priv)) {
 387                 /*
 388                  * FIXME can't find a bit like this for 915G, and
 389                  * and yet it does have the related watermark in
 390                  * FW_BLC_SELF. What's going on?
 391                  */
 392                 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
 393                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 394                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 395                 I915_WRITE(INSTPM, val);
 396                 POSTING_READ(INSTPM);
 397         } else {
 398                 return false;
 399         }
 400
 401         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 402
 403         DRM_DEBUG_KMS("memory self-refresh is %s (was %s)\n",
 404                       enableddisabled(enable),
 405                       enableddisabled(was_enabled));
 406
 407         return was_enabled;
 408 }
 409
 410 /**
 411  * intel_set_memory_cxsr - Configure CxSR state
 412  * @dev_priv: i915 device
 413  * @enable: Allow vs. disallow CxSR
 414  *
 415  * Allow or disallow the system to enter a special CxSR
 416  * (C-state self refresh) state. What typically happens in CxSR mode
 417  * is that several display FIFOs may get combined into a single larger
 418  * FIFO for a particular plane (so called max FIFO mode) to allow the
 419  * system to defer memory fetches longer, and the memory will enter
 420  * self refresh.
 421  *
 422  * Note that enabling CxSR does not guarantee that the system enter
 423  * this special mode, nor does it guarantee that the system stays
 424  * in that mode once entered. So this just allows/disallows the system
 425  * to autonomously utilize the CxSR mode. Other factors such as core
 426  * C-states will affect when/if the system actually enters/exits the
 427  * CxSR mode.
 428  *
 429  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 430  * and the system is free to enter/exit memory self refresh at any time
 431  * even when the use of CxSR has been disallowed.
 432  *
 433  * While the system is actually in the CxSR/max FIFO mode, some plane
 434  * control registers will not get latched on vblank. Thus in order to
 435  * guarantee the system will respond to changes in the plane registers
 436  * we must always disallow CxSR prior to making changes to those registers.
 437  * Unfortunately the system will re-evaluate the CxSR conditions at
 438  * frame start which happens after vblank start (which is when the plane
 439  * registers would get latched), so we can't proceed with the plane update
 440  * during the same frame where we disallowed CxSR.
 441  *
 442  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 443  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 444  * the hardware w.r.t. HPLL SR when writing to plane registers.
 445  * Disallowing just CxSR is sufficient.
 446  */
 447 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 448 {
 449         bool ret;
 450
 451         mutex_lock(&dev_priv->wm.wm_mutex);
 452         ret = _intel_set_memory_cxsr(dev_priv, enable);
 453         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 454                 dev_priv->wm.vlv.cxsr = enable;
 455         else if (IS_G4X(dev_priv))
 456                 dev_priv->wm.g4x.cxsr = enable;
 457         mutex_unlock(&dev_priv->wm.wm_mutex);
 458
 459         return ret;
 460 }
 461
 462 /*
 463  * Latency for FIFO fetches is dependent on several factors:
 464  *   - memory configuration (speed, channels)
 465  *   - chipset
 466  *   - current MCH state
 467  * It can be fairly high in some situations, so here we assume a fairly
 468  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 469  * set this value too high, the FIFO will fetch frequently to stay full)
 470  * and power consumption (set it too low to save power and we might see
 471  * FIFO underruns and display "flicker").
 472  *
 473  * A value of 5us seems to be a good balance; safe for very low end
 474  * platforms but not overly aggressive on lower latency configs.
 475  */
 476 static const int pessimal_latency_ns = 5000;
 477
 478 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 479         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 480
 481 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 482 {
 483         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 484         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 485         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 486         enum pipe pipe = crtc->pipe;
 487         int sprite0_start, sprite1_start;
 488
 489         switch (pipe) {
 490                 uint32_t dsparb, dsparb2, dsparb3;
 491         case PIPE_A:
 492                 dsparb = I915_READ(DSPARB);
 493                 dsparb2 = I915_READ(DSPARB2);
 494                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 495                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 496                 break;
 497         case PIPE_B:
 498                 dsparb = I915_READ(DSPARB);
 499                 dsparb2 = I915_READ(DSPARB2);
 500                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 501                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 502                 break;
 503         case PIPE_C:
 504                 dsparb2 = I915_READ(DSPARB2);
 505                 dsparb3 = I915_READ(DSPARB3);
 506                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 507                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 508                 break;
 509         default:
 510                 MISSING_CASE(pipe);
 511                 return;
 512         }
 513
 514         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 515         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 516         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 517         fifo_state->plane[PLANE_CURSOR] = 63;
 518 }
 519
 520 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 521                               enum i9xx_plane_id i9xx_plane)
 522 {
 523         uint32_t dsparb = I915_READ(DSPARB);
 524         int size;
 525
 526         size = dsparb & 0x7f;
 527         if (i9xx_plane == PLANE_B)
 528                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 529
 530         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 531                       dsparb, plane_name(i9xx_plane), size);
 532
 533         return size;
 534 }
 535
 536 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 537                               enum i9xx_plane_id i9xx_plane)
 538 {
 539         uint32_t dsparb = I915_READ(DSPARB);
 540         int size;
 541
 542         size = dsparb & 0x1ff;
 543         if (i9xx_plane == PLANE_B)
 544                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 545         size >>= 1; /* Convert to cachelines */
 546
 547         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 548                       dsparb, plane_name(i9xx_plane), size);
 549
 550         return size;
 551 }
 552
 553 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 554                               enum i9xx_plane_id i9xx_plane)
 555 {
 556         uint32_t dsparb = I915_READ(DSPARB);
 557         int size;
 558
 559         size = dsparb & 0x7f;
 560         size >>= 2; /* Convert to cachelines */
 561
 562         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 563                       dsparb, plane_name(i9xx_plane), size);
 564
 565         return size;
 566 }
 567
 568 /* Pineview has different values for various configs */
 569 static const struct intel_watermark_params pineview_display_wm = {
 570         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 571         .max_wm = PINEVIEW_MAX_WM,
 572         .default_wm = PINEVIEW_DFT_WM,
 573         .guard_size = PINEVIEW_GUARD_WM,
 574         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 575 };
 576 static const struct intel_watermark_params pineview_display_hplloff_wm = {
 577         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 578         .max_wm = PINEVIEW_MAX_WM,
 579         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 580         .guard_size = PINEVIEW_GUARD_WM,
 581         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 582 };
 583 static const struct intel_watermark_params pineview_cursor_wm = {
 584         .fifo_size = PINEVIEW_CURSOR_FIFO,
 585         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 586         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 587         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 588         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 589 };
 590 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
 591         .fifo_size = PINEVIEW_CURSOR_FIFO,
 592         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 593         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 594         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 595         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 596 };
 597 static const struct intel_watermark_params i965_cursor_wm_info = {
 598         .fifo_size = I965_CURSOR_FIFO,
 599         .max_wm = I965_CURSOR_MAX_WM,
 600         .default_wm = I965_CURSOR_DFT_WM,
 601         .guard_size = 2,
 602         .cacheline_size = I915_FIFO_LINE_SIZE,
 603 };
 604 static const struct intel_watermark_params i945_wm_info = {
 605         .fifo_size = I945_FIFO_SIZE,
 606         .max_wm = I915_MAX_WM,
 607         .default_wm = 1,
 608         .guard_size = 2,
 609         .cacheline_size = I915_FIFO_LINE_SIZE,
 610 };
 611 static const struct intel_watermark_params i915_wm_info = {
 612         .fifo_size = I915_FIFO_SIZE,
 613         .max_wm = I915_MAX_WM,
 614         .default_wm = 1,
 615         .guard_size = 2,
 616         .cacheline_size = I915_FIFO_LINE_SIZE,
 617 };
 618 static const struct intel_watermark_params i830_a_wm_info = {
 619         .fifo_size = I855GM_FIFO_SIZE,
 620         .max_wm = I915_MAX_WM,
 621         .default_wm = 1,
 622         .guard_size = 2,
 623         .cacheline_size = I830_FIFO_LINE_SIZE,
 624 };
 625 static const struct intel_watermark_params i830_bc_wm_info = {
 626         .fifo_size = I855GM_FIFO_SIZE,
 627         .max_wm = I915_MAX_WM/2,
 628         .default_wm = 1,
 629         .guard_size = 2,
 630         .cacheline_size = I830_FIFO_LINE_SIZE,
 631 };
 632 static const struct intel_watermark_params i845_wm_info = {
 633         .fifo_size = I830_FIFO_SIZE,
 634         .max_wm = I915_MAX_WM,
 635         .default_wm = 1,
 636         .guard_size = 2,
 637         .cacheline_size = I830_FIFO_LINE_SIZE,
 638 };
 639
 640 /**
 641  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 642  * @pixel_rate: Pipe pixel rate in kHz
 643  * @cpp: Plane bytes per pixel
 644  * @latency: Memory wakeup latency in 0.1us units
 645  *
 646  * Compute the watermark using the method 1 or "small buffer"
 647  * formula. The caller may additonally add extra cachelines
 648  * to account for TLB misses and clock crossings.
 649  *
 650  * This method is concerned with the short term drain rate
 651  * of the FIFO, ie. it does not account for blanking periods
 652  * which would effectively reduce the average drain rate across
 653  * a longer period. The name "small" refers to the fact the
 654  * FIFO is relatively small compared to the amount of data
 655  * fetched.
 656  *
 657  * The FIFO level vs. time graph might look something like:
 658  *
 659  *   |\   |\
 660  *   | \  | \
 661  * __---__---__ (- plane active, _ blanking)
 662  * -> time
 663  *
 664  * or perhaps like this:
 665  *
 666  *   |\|\  |\|\
 667  * __----__----__ (- plane active, _ blanking)
 668  * -> time
 669  *
 670  * Returns:
 671  * The watermark in bytes
 672  */
 673 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 674                                      unsigned int cpp,
 675                                      unsigned int latency)
 676 {
 677         uint64_t ret;
 678
 679         ret = (uint64_t) pixel_rate * cpp * latency;
 680         ret = DIV_ROUND_UP_ULL(ret, 10000);
 681
 682         return ret;
 683 }
 684
 685 /**
 686  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 687  * @pixel_rate: Pipe pixel rate in kHz
 688  * @htotal: Pipe horizontal total
 689  * @width: Plane width in pixels
 690  * @cpp: Plane bytes per pixel
 691  * @latency: Memory wakeup latency in 0.1us units
 692  *
 693  * Compute the watermark using the method 2 or "large buffer"
 694  * formula. The caller may additonally add extra cachelines
 695  * to account for TLB misses and clock crossings.
 696  *
 697  * This method is concerned with the long term drain rate
 698  * of the FIFO, ie. it does account for blanking periods
 699  * which effectively reduce the average drain rate across
 700  * a longer period. The name "large" refers to the fact the
 701  * FIFO is relatively large compared to the amount of data
 702  * fetched.
 703  *
 704  * The FIFO level vs. time graph might look something like:
 705  *
 706  *    |\___       |\___
 707  *    |    \___   |    \___
 708  *    |        \  |        \
 709  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 710  * -> time
 711  *
 712  * Returns:
 713  * The watermark in bytes
 714  */
 715 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 716                                      unsigned int htotal,
 717                                      unsigned int width,
 718                                      unsigned int cpp,
 719                                      unsigned int latency)
 720 {
 721         unsigned int ret;
 722
 723         /*
 724          * FIXME remove once all users are computing
 725          * watermarks in the correct place.
 726          */
 727         if (WARN_ON_ONCE(htotal == 0))
 728                 htotal = 1;
 729
 730         ret = (latency * pixel_rate) / (htotal * 10000);
 731         ret = (ret + 1) * width * cpp;
 732
 733         return ret;
 734 }
 735
 736 /**
 737  * intel_calculate_wm - calculate watermark level
 738  * @pixel_rate: pixel clock
 739  * @wm: chip FIFO params
 740  * @fifo_size: size of the FIFO buffer
 741  * @cpp: bytes per pixel
 742  * @latency_ns: memory latency for the platform
 743  *
 744  * Calculate the watermark level (the level at which the display plane will
 745  * start fetching from memory again).  Each chip has a different display
 746  * FIFO size and allocation, so the caller needs to figure that out and pass
 747  * in the correct intel_watermark_params structure.
 748  *
 749  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 750  * on the pixel size.  When it reaches the watermark level, it'll start
 751  * fetching FIFO line sized based chunks from memory until the FIFO fills
 752  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 753  * will occur, and a display engine hang could result.
 754  */
 755 static unsigned int intel_calculate_wm(int pixel_rate,
 756                                        const struct intel_watermark_params *wm,
 757                                        int fifo_size, int cpp,
 758                                        unsigned int latency_ns)
 759 {
 760         int entries, wm_size;
 761
 762         /*
 763          * Note: we need to make sure we don't overflow for various clock &
 764          * latency values.
 765          * clocks go from a few thousand to several hundred thousand.
 766          * latency is usually a few thousand
 767          */
 768         entries = intel_wm_method1(pixel_rate, cpp,
 769                                    latency_ns / 100);
 770         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 771                 wm->guard_size;
 772         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 773
 774         wm_size = fifo_size - entries;
 775         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 776
 777         /* Don't promote wm_size to unsigned... */
 778         if (wm_size > wm->max_wm)
 779                 wm_size = wm->max_wm;
 780         if (wm_size <= 0)
 781                 wm_size = wm->default_wm;
 782
 783         /*
 784          * Bspec seems to indicate that the value shouldn't be lower than
 785          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 786          * Lets go for 8 which is the burst size since certain platforms
 787          * already use a hardcoded 8 (which is what the spec says should be
 788          * done).
 789          */
 790         if (wm_size <= 8)
 791                 wm_size = 8;
 792
 793         return wm_size;
 794 }
 795
 796 static bool is_disabling(int old, int new, int threshold)
 797 {
 798         return old >= threshold && new < threshold;
 799 }
 800
 801 static bool is_enabling(int old, int new, int threshold)
 802 {
 803         return old < threshold && new >= threshold;
 804 }
 805
 806 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 807 {
 808         return dev_priv->wm.max_level + 1;
 809 }
 810
 811 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 812                                    const struct intel_plane_state *plane_state)
 813 {
 814         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
 815
 816         /* FIXME check the 'enable' instead */
 817         if (!crtc_state->base.active)
 818                 return false;
 819
 820         /*
 821          * Treat cursor with fb as always visible since cursor updates
 822          * can happen faster than the vrefresh rate, and the current
 823          * watermark code doesn't handle that correctly. Cursor updates
 824          * which set/clear the fb or change the cursor size are going
 825          * to get throttled by intel_legacy_cursor_update() to work
 826          * around this problem with the watermark code.
 827          */
 828         if (plane->id == PLANE_CURSOR)
 829                 return plane_state->base.fb != NULL;
 830         else
 831                 return plane_state->base.visible;
 832 }
 833
 834 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 835 {
 836         struct intel_crtc *crtc, *enabled = NULL;
 837
 838         for_each_intel_crtc(&dev_priv->drm, crtc) {
 839                 if (intel_crtc_active(crtc)) {
 840                         if (enabled)
 841                                 return NULL;
 842                         enabled = crtc;
 843                 }
 844         }
 845
 846         return enabled;
 847 }
 848
 849 static void pineview_update_wm(struct intel_crtc *unused_crtc)
 850 {
 851         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
 852         struct intel_crtc *crtc;
 853         const struct cxsr_latency *latency;
 854         u32 reg;
 855         unsigned int wm;
 856
 857         latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
 858                                          dev_priv->is_ddr3,
 859                                          dev_priv->fsb_freq,
 860                                          dev_priv->mem_freq);
 861         if (!latency) {
 862                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 863                 intel_set_memory_cxsr(dev_priv, false);
 864                 return;
 865         }
 866
 867         crtc = single_enabled_crtc(dev_priv);
 868         if (crtc) {
 869                 const struct drm_display_mode *adjusted_mode =
 870                         &crtc->config->base.adjusted_mode;
 871                 const struct drm_framebuffer *fb =
 872                         crtc->base.primary->state->fb;
 873                 int cpp = fb->format->cpp[0];
 874                 int clock = adjusted_mode->crtc_clock;
 875
 876                 /* Display SR */
 877                 wm = intel_calculate_wm(clock, &pineview_display_wm,
 878                                         pineview_display_wm.fifo_size,
 879                                         cpp, latency->display_sr);
 880                 reg = I915_READ(DSPFW1);
 881                 reg &= ~DSPFW_SR_MASK;
 882                 reg |= FW_WM(wm, SR);
 883                 I915_WRITE(DSPFW1, reg);
 884                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
 885
 886                 /* cursor SR */
 887                 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
 888                                         pineview_display_wm.fifo_size,
 889                                         4, latency->cursor_sr);
 890                 reg = I915_READ(DSPFW3);
 891                 reg &= ~DSPFW_CURSOR_SR_MASK;
 892                 reg |= FW_WM(wm, CURSOR_SR);
 893                 I915_WRITE(DSPFW3, reg);
 894
 895                 /* Display HPLL off SR */
 896                 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
 897                                         pineview_display_hplloff_wm.fifo_size,
 898                                         cpp, latency->display_hpll_disable);
 899                 reg = I915_READ(DSPFW3);
 900                 reg &= ~DSPFW_HPLL_SR_MASK;
 901                 reg |= FW_WM(wm, HPLL_SR);
 902                 I915_WRITE(DSPFW3, reg);
 903
 904                 /* cursor HPLL off SR */
 905                 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
 906                                         pineview_display_hplloff_wm.fifo_size,
 907                                         4, latency->cursor_hpll_disable);
 908                 reg = I915_READ(DSPFW3);
 909                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 910                 reg |= FW_WM(wm, HPLL_CURSOR);
 911                 I915_WRITE(DSPFW3, reg);
 912                 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
 913
 914                 intel_set_memory_cxsr(dev_priv, true);
 915         } else {
 916                 intel_set_memory_cxsr(dev_priv, false);
 917         }
 918 }
 919
 920 /*
 921  * Documentation says:
 922  * "If the line size is small, the TLB fetches can get in the way of the
 923  *  data fetches, causing some lag in the pixel data return which is not
 924  *  accounted for in the above formulas. The following adjustment only
 925  *  needs to be applied if eight whole lines fit in the buffer at once.
 926  *  The WM is adjusted upwards by the difference between the FIFO size
 927  *  and the size of 8 whole lines. This adjustment is always performed
 928  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 929  */
 930 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 931 {
 932         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 933
 934         return max(0, tlb_miss);
 935 }
 936
 937 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 938                                 const struct g4x_wm_values *wm)
 939 {
 940         enum pipe pipe;
 941
 942         for_each_pipe(dev_priv, pipe)
 943                 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 944
 945         I915_WRITE(DSPFW1,
 946                    FW_WM(wm->sr.plane, SR) |
 947                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 948                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 949                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 950         I915_WRITE(DSPFW2,
 951                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 952                    FW_WM(wm->sr.fbc, FBC_SR) |
 953                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 954                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 955                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 956                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 957         I915_WRITE(DSPFW3,
 958                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 959                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 960                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 961                    FW_WM(wm->hpll.plane, HPLL_SR));
 962
 963         POSTING_READ(DSPFW1);
 964 }
 965
 966 #define FW_WM_VLV(value, plane) \
 967         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 968
 969 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 970                                 const struct vlv_wm_values *wm)
 971 {
 972         enum pipe pipe;
 973
 974         for_each_pipe(dev_priv, pipe) {
 975                 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 976
 977                 I915_WRITE(VLV_DDL(pipe),
 978                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 979                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 980                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 981                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 982         }
 983
 984         /*
 985          * Zero the (unused) WM1 watermarks, and also clear all the
 986          * high order bits so that there are no out of bounds values
 987          * present in the registers during the reprogramming.
 988          */
 989         I915_WRITE(DSPHOWM, 0);
 990         I915_WRITE(DSPHOWM1, 0);
 991         I915_WRITE(DSPFW4, 0);
 992         I915_WRITE(DSPFW5, 0);
 993         I915_WRITE(DSPFW6, 0);
 994
 995         I915_WRITE(DSPFW1,
 996                    FW_WM(wm->sr.plane, SR) |
 997                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 998                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 999                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1000         I915_WRITE(DSPFW2,
1001                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1002                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1003                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1004         I915_WRITE(DSPFW3,
1005                    FW_WM(wm->sr.cursor, CURSOR_SR));
1006
1007         if (IS_CHERRYVIEW(dev_priv)) {
1008                 I915_WRITE(DSPFW7_CHV,
1009                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1010                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1011                 I915_WRITE(DSPFW8_CHV,
1012                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1013                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1014                 I915_WRITE(DSPFW9_CHV,
1015                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1016                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1017                 I915_WRITE(DSPHOWM,
1018                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1019                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1020                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1021                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1022                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1023                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1024                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1025                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1026                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1027                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1028         } else {
1029                 I915_WRITE(DSPFW7,
1030                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1031                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1032                 I915_WRITE(DSPHOWM,
1033                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1034                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1035                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1036                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1037                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1038                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1039                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1040         }
1041
1042         POSTING_READ(DSPFW1);
1043 }
1044
1045 #undef FW_WM_VLV
1046
1047 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1048 {
1049         /* all latencies in usec */
1050         dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1051         dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1052         dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1053
1054         dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1055 }
1056
1057 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1058 {
1059         /*
1060          * DSPCNTR[13] supposedly controls whether the
1061          * primary plane can use the FIFO space otherwise
1062          * reserved for the sprite plane. It's not 100% clear
1063          * what the actual FIFO size is, but it looks like we
1064          * can happily set both primary and sprite watermarks
1065          * up to 127 cachelines. So that would seem to mean
1066          * that either DSPCNTR[13] doesn't do anything, or that
1067          * the total FIFO is >= 256 cachelines in size. Either
1068          * way, we don't seem to have to worry about this
1069          * repartitioning as the maximum watermark value the
1070          * register can hold for each plane is lower than the
1071          * minimum FIFO size.
1072          */
1073         switch (plane_id) {
1074         case PLANE_CURSOR:
1075                 return 63;
1076         case PLANE_PRIMARY:
1077                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1078         case PLANE_SPRITE0:
1079                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1080         default:
1081                 MISSING_CASE(plane_id);
1082                 return 0;
1083         }
1084 }
1085
1086 static int g4x_fbc_fifo_size(int level)
1087 {
1088         switch (level) {
1089         case G4X_WM_LEVEL_SR:
1090                 return 7;
1091         case G4X_WM_LEVEL_HPLL:
1092                 return 15;
1093         default:
1094                 MISSING_CASE(level);
1095                 return 0;
1096         }
1097 }
1098
1099 static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1100                                const struct intel_plane_state *plane_state,
1101                                int level)
1102 {
1103         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1104         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1105         const struct drm_display_mode *adjusted_mode =
1106                 &crtc_state->base.adjusted_mode;
1107         unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1108         unsigned int clock, htotal, cpp, width, wm;
1109
1110         if (latency == 0)
1111                 return USHRT_MAX;
1112
1113         if (!intel_wm_plane_visible(crtc_state, plane_state))
1114                 return 0;
1115
1116         /*
1117          * Not 100% sure which way ELK should go here as the
1118          * spec only says CL/CTG should assume 32bpp and BW
1119          * doesn't need to. But as these things followed the
1120          * mobile vs. desktop lines on gen3 as well, let's
1121          * assume ELK doesn't need this.
1122          *
1123          * The spec also fails to list such a restriction for
1124          * the HPLL watermark, which seems a little strange.
1125          * Let's use 32bpp for the HPLL watermark as well.
1126          */
1127         if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1128             level != G4X_WM_LEVEL_NORMAL)
1129                 cpp = 4;
1130         else
1131                 cpp = plane_state->base.fb->format->cpp[0];
1132
1133         clock = adjusted_mode->crtc_clock;
1134         htotal = adjusted_mode->crtc_htotal;
1135
1136         if (plane->id == PLANE_CURSOR)
1137                 width = plane_state->base.crtc_w;
1138         else
1139                 width = drm_rect_width(&plane_state->base.dst);
1140
1141         if (plane->id == PLANE_CURSOR) {
1142                 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1143         } else if (plane->id == PLANE_PRIMARY &&
1144                    level == G4X_WM_LEVEL_NORMAL) {
1145                 wm = intel_wm_method1(clock, cpp, latency);
1146         } else {
1147                 unsigned int small, large;
1148
1149                 small = intel_wm_method1(clock, cpp, latency);
1150                 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1151
1152                 wm = min(small, large);
1153         }
1154
1155         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1156                               width, cpp);
1157
1158         wm = DIV_ROUND_UP(wm, 64) + 2;
1159
1160         return min_t(unsigned int, wm, USHRT_MAX);
1161 }
1162
1163 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1164                                  int level, enum plane_id plane_id, u16 value)
1165 {
1166         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1167         bool dirty = false;
1168
1169         for (; level < intel_wm_num_levels(dev_priv); level++) {
1170                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1171
1172                 dirty |= raw->plane[plane_id] != value;
1173                 raw->plane[plane_id] = value;
1174         }
1175
1176         return dirty;
1177 }
1178
1179 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1180                                int level, u16 value)
1181 {
1182         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1183         bool dirty = false;
1184
1185         /* NORMAL level doesn't have an FBC watermark */
1186         level = max(level, G4X_WM_LEVEL_SR);
1187
1188         for (; level < intel_wm_num_levels(dev_priv); level++) {
1189                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1190
1191                 dirty |= raw->fbc != value;
1192                 raw->fbc = value;
1193         }
1194
1195         return dirty;
1196 }
1197
1198 static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
1199                                    const struct intel_plane_state *pstate,
1200                                    uint32_t pri_val);
1201
1202 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1203                                      const struct intel_plane_state *plane_state)
1204 {
1205         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1206         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1207         enum plane_id plane_id = plane->id;
1208         bool dirty = false;
1209         int level;
1210
1211         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1212                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1213                 if (plane_id == PLANE_PRIMARY)
1214                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1215                 goto out;
1216         }
1217
1218         for (level = 0; level < num_levels; level++) {
1219                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1220                 int wm, max_wm;
1221
1222                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1223                 max_wm = g4x_plane_fifo_size(plane_id, level);
1224
1225                 if (wm > max_wm)
1226                         break;
1227
1228                 dirty |= raw->plane[plane_id] != wm;
1229                 raw->plane[plane_id] = wm;
1230
1231                 if (plane_id != PLANE_PRIMARY ||
1232                     level == G4X_WM_LEVEL_NORMAL)
1233                         continue;
1234
1235                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1236                                         raw->plane[plane_id]);
1237                 max_wm = g4x_fbc_fifo_size(level);
1238
1239                 /*
1240                  * FBC wm is not mandatory as we
1241                  * can always just disable its use.
1242                  */
1243                 if (wm > max_wm)
1244                         wm = USHRT_MAX;
1245
1246                 dirty |= raw->fbc != wm;
1247                 raw->fbc = wm;
1248         }
1249
1250         /* mark watermarks as invalid */
1251         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1252
1253         if (plane_id == PLANE_PRIMARY)
1254                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1255
1256  out:
1257         if (dirty) {
1258                 DRM_DEBUG_KMS("%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1259                               plane->base.name,
1260                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1261                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1262                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1263
1264                 if (plane_id == PLANE_PRIMARY)
1265                         DRM_DEBUG_KMS("FBC watermarks: SR=%d, HPLL=%d\n",
1266                                       crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1267                                       crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1268         }
1269
1270         return dirty;
1271 }
1272
1273 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1274                                       enum plane_id plane_id, int level)
1275 {
1276         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1277
1278         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1279 }
1280
1281 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1282                                      int level)
1283 {
1284         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1285
1286         if (level > dev_priv->wm.max_level)
1287                 return false;
1288
1289         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1290                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1291                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1292 }
1293
1294 /* mark all levels starting from 'level' as invalid */
1295 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1296                                struct g4x_wm_state *wm_state, int level)
1297 {
1298         if (level <= G4X_WM_LEVEL_NORMAL) {
1299                 enum plane_id plane_id;
1300
1301                 for_each_plane_id_on_crtc(crtc, plane_id)
1302                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1303         }
1304
1305         if (level <= G4X_WM_LEVEL_SR) {
1306                 wm_state->cxsr = false;
1307                 wm_state->sr.cursor = USHRT_MAX;
1308                 wm_state->sr.plane = USHRT_MAX;
1309                 wm_state->sr.fbc = USHRT_MAX;
1310         }
1311
1312         if (level <= G4X_WM_LEVEL_HPLL) {
1313                 wm_state->hpll_en = false;
1314                 wm_state->hpll.cursor = USHRT_MAX;
1315                 wm_state->hpll.plane = USHRT_MAX;
1316                 wm_state->hpll.fbc = USHRT_MAX;
1317         }
1318 }
1319
1320 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1321 {
1322         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1323         struct intel_atomic_state *state =
1324                 to_intel_atomic_state(crtc_state->base.state);
1325         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1326         int num_active_planes = hweight32(crtc_state->active_planes &
1327                                           ~BIT(PLANE_CURSOR));
1328         const struct g4x_pipe_wm *raw;
1329         const struct intel_plane_state *old_plane_state;
1330         const struct intel_plane_state *new_plane_state;
1331         struct intel_plane *plane;
1332         enum plane_id plane_id;
1333         int i, level;
1334         unsigned int dirty = 0;
1335
1336         for_each_oldnew_intel_plane_in_state(state, plane,
1337                                              old_plane_state,
1338                                              new_plane_state, i) {
1339                 if (new_plane_state->base.crtc != &crtc->base &&
1340                     old_plane_state->base.crtc != &crtc->base)
1341                         continue;
1342
1343                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1344                         dirty |= BIT(plane->id);
1345         }
1346
1347         if (!dirty)
1348                 return 0;
1349
1350         level = G4X_WM_LEVEL_NORMAL;
1351         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1352                 goto out;
1353
1354         raw = &crtc_state->wm.g4x.raw[level];
1355         for_each_plane_id_on_crtc(crtc, plane_id)
1356                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1357
1358         level = G4X_WM_LEVEL_SR;
1359
1360         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1361                 goto out;
1362
1363         raw = &crtc_state->wm.g4x.raw[level];
1364         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1365         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1366         wm_state->sr.fbc = raw->fbc;
1367
1368         wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1369
1370         level = G4X_WM_LEVEL_HPLL;
1371
1372         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1373                 goto out;
1374
1375         raw = &crtc_state->wm.g4x.raw[level];
1376         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1377         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1378         wm_state->hpll.fbc = raw->fbc;
1379
1380         wm_state->hpll_en = wm_state->cxsr;
1381
1382         level++;
1383
1384  out:
1385         if (level == G4X_WM_LEVEL_NORMAL)
1386                 return -EINVAL;
1387
1388         /* invalidate the higher levels */
1389         g4x_invalidate_wms(crtc, wm_state, level);
1390
1391         /*
1392          * Determine if the FBC watermark(s) can be used. IF
1393          * this isn't the case we prefer to disable the FBC
1394          ( watermark(s) rather than disable the SR/HPLL
1395          * level(s) entirely.
1396          */
1397         wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;
1398
1399         if (level >= G4X_WM_LEVEL_SR &&
1400             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1401                 wm_state->fbc_en = false;
1402         else if (level >= G4X_WM_LEVEL_HPLL &&
1403                  wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1404                 wm_state->fbc_en = false;
1405
1406         return 0;
1407 }
1408
1409 static int g4x_compute_intermediate_wm(struct drm_device *dev,
1410                                        struct intel_crtc *crtc,
1411                                        struct intel_crtc_state *new_crtc_state)
1412 {
1413         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1414         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1415         struct intel_atomic_state *intel_state =
1416                 to_intel_atomic_state(new_crtc_state->base.state);
1417         const struct intel_crtc_state *old_crtc_state =
1418                 intel_atomic_get_old_crtc_state(intel_state, crtc);
1419         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1420         enum plane_id plane_id;
1421
1422         if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
1423                 *intermediate = *optimal;
1424
1425                 intermediate->cxsr = false;
1426                 intermediate->hpll_en = false;
1427                 goto out;
1428         }
1429
1430         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1431                 !new_crtc_state->disable_cxsr;
1432         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1433                 !new_crtc_state->disable_cxsr;
1434         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1435
1436         for_each_plane_id_on_crtc(crtc, plane_id) {
1437                 intermediate->wm.plane[plane_id] =
1438                         max(optimal->wm.plane[plane_id],
1439                             active->wm.plane[plane_id]);
1440
1441                 WARN_ON(intermediate->wm.plane[plane_id] >
1442                         g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1443         }
1444
1445         intermediate->sr.plane = max(optimal->sr.plane,
1446                                      active->sr.plane);
1447         intermediate->sr.cursor = max(optimal->sr.cursor,
1448                                       active->sr.cursor);
1449         intermediate->sr.fbc = max(optimal->sr.fbc,
1450                                    active->sr.fbc);
1451
1452         intermediate->hpll.plane = max(optimal->hpll.plane,
1453                                        active->hpll.plane);
1454         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1455                                         active->hpll.cursor);
1456         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1457                                      active->hpll.fbc);
1458
1459         WARN_ON((intermediate->sr.plane >
1460                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1461                  intermediate->sr.cursor >
1462                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1463                 intermediate->cxsr);
1464         WARN_ON((intermediate->sr.plane >
1465                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1466                  intermediate->sr.cursor >
1467                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1468                 intermediate->hpll_en);
1469
1470         WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1471                 intermediate->fbc_en && intermediate->cxsr);
1472         WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1473                 intermediate->fbc_en && intermediate->hpll_en);
1474
1475 out:
1476         /*
1477          * If our intermediate WM are identical to the final WM, then we can
1478          * omit the post-vblank programming; only update if it's different.
1479          */
1480         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1481                 new_crtc_state->wm.need_postvbl_update = true;
1482
1483         return 0;
1484 }
1485
1486 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1487                          struct g4x_wm_values *wm)
1488 {
1489         struct intel_crtc *crtc;
1490         int num_active_crtcs = 0;
1491
1492         wm->cxsr = true;
1493         wm->hpll_en = true;
1494         wm->fbc_en = true;
1495
1496         for_each_intel_crtc(&dev_priv->drm, crtc) {
1497                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1498
1499                 if (!crtc->active)
1500                         continue;
1501
1502                 if (!wm_state->cxsr)
1503                         wm->cxsr = false;
1504                 if (!wm_state->hpll_en)
1505                         wm->hpll_en = false;
1506                 if (!wm_state->fbc_en)
1507                         wm->fbc_en = false;
1508
1509                 num_active_crtcs++;
1510         }
1511
1512         if (num_active_crtcs != 1) {
1513                 wm->cxsr = false;
1514                 wm->hpll_en = false;
1515                 wm->fbc_en = false;
1516         }
1517
1518         for_each_intel_crtc(&dev_priv->drm, crtc) {
1519                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1520                 enum pipe pipe = crtc->pipe;
1521
1522                 wm->pipe[pipe] = wm_state->wm;
1523                 if (crtc->active && wm->cxsr)
1524                         wm->sr = wm_state->sr;
1525                 if (crtc->active && wm->hpll_en)
1526                         wm->hpll = wm_state->hpll;
1527         }
1528 }
1529
1530 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1531 {
1532         struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1533         struct g4x_wm_values new_wm = {};
1534
1535         g4x_merge_wm(dev_priv, &new_wm);
1536
1537         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1538                 return;
1539
1540         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1541                 _intel_set_memory_cxsr(dev_priv, false);
1542
1543         g4x_write_wm_values(dev_priv, &new_wm);
1544
1545         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1546                 _intel_set_memory_cxsr(dev_priv, true);
1547
1548         *old_wm = new_wm;
1549 }
1550
1551 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1552                                    struct intel_crtc_state *crtc_state)
1553 {
1554         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1555         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1556
1557         mutex_lock(&dev_priv->wm.wm_mutex);
1558         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1559         g4x_program_watermarks(dev_priv);
1560         mutex_unlock(&dev_priv->wm.wm_mutex);
1561 }
1562
1563 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1564                                     struct intel_crtc_state *crtc_state)
1565 {
1566         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1567         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
1568
1569         if (!crtc_state->wm.need_postvbl_update)
1570                 return;
1571
1572         mutex_lock(&dev_priv->wm.wm_mutex);
1573         intel_crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1574         g4x_program_watermarks(dev_priv);
1575         mutex_unlock(&dev_priv->wm.wm_mutex);
1576 }
1577
1578 /* latency must be in 0.1us units. */
1579 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1580                                    unsigned int htotal,
1581                                    unsigned int width,
1582                                    unsigned int cpp,
1583                                    unsigned int latency)
1584 {
1585         unsigned int ret;
1586
1587         ret = intel_wm_method2(pixel_rate, htotal,
1588                                width, cpp, latency);
1589         ret = DIV_ROUND_UP(ret, 64);
1590
1591         return ret;
1592 }
1593
1594 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1595 {
1596         /* all latencies in usec */
1597         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1598
1599         dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1600
1601         if (IS_CHERRYVIEW(dev_priv)) {
1602                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1603                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1604
1605                 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1606         }
1607 }
1608
1609 static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1610                                      const struct intel_plane_state *plane_state,
1611                                      int level)
1612 {
1613         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1614         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1615         const struct drm_display_mode *adjusted_mode =
1616                 &crtc_state->base.adjusted_mode;
1617         unsigned int clock, htotal, cpp, width, wm;
1618
1619         if (dev_priv->wm.pri_latency[level] == 0)
1620                 return USHRT_MAX;
1621
1622         if (!intel_wm_plane_visible(crtc_state, plane_state))
1623                 return 0;
1624
1625         cpp = plane_state->base.fb->format->cpp[0];
1626         clock = adjusted_mode->crtc_clock;
1627         htotal = adjusted_mode->crtc_htotal;
1628         width = crtc_state->pipe_src_w;
1629
1630         if (plane->id == PLANE_CURSOR) {
1631                 /*
1632                  * FIXME the formula gives values that are
1633                  * too big for the cursor FIFO, and hence we
1634                  * would never be able to use cursors. For
1635                  * now just hardcode the watermark.
1636                  */
1637                 wm = 63;
1638         } else {
1639                 wm = vlv_wm_method2(clock, htotal, width, cpp,
1640                                     dev_priv->wm.pri_latency[level] * 10);
1641         }
1642
1643         return min_t(unsigned int, wm, USHRT_MAX);
1644 }
1645
1646 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1647 {
1648         return (active_planes & (BIT(PLANE_SPRITE0) |
1649                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1650 }
1651
1652 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1653 {
1654         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1655         const struct g4x_pipe_wm *raw =
1656                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1657         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1658         unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1659         int num_active_planes = hweight32(active_planes);
1660         const int fifo_size = 511;
1661         int fifo_extra, fifo_left = fifo_size;
1662         int sprite0_fifo_extra = 0;
1663         unsigned int total_rate;
1664         enum plane_id plane_id;
1665
1666         /*
1667          * When enabling sprite0 after sprite1 has already been enabled
1668          * we tend to get an underrun unless sprite0 already has some
1669          * FIFO space allcoated. Hence we always allocate at least one
1670          * cacheline for sprite0 whenever sprite1 is enabled.
1671          *
1672          * All other plane enable sequences appear immune to this problem.
1673          */
1674         if (vlv_need_sprite0_fifo_workaround(active_planes))
1675                 sprite0_fifo_extra = 1;
1676
1677         total_rate = raw->plane[PLANE_PRIMARY] +
1678                 raw->plane[PLANE_SPRITE0] +
1679                 raw->plane[PLANE_SPRITE1] +
1680                 sprite0_fifo_extra;
1681
1682         if (total_rate > fifo_size)
1683                 return -EINVAL;
1684
1685         if (total_rate == 0)
1686                 total_rate = 1;
1687
1688         for_each_plane_id_on_crtc(crtc, plane_id) {
1689                 unsigned int rate;
1690
1691                 if ((active_planes & BIT(plane_id)) == 0) {
1692                         fifo_state->plane[plane_id] = 0;
1693                         continue;
1694                 }
1695
1696                 rate = raw->plane[plane_id];
1697                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1698                 fifo_left -= fifo_state->plane[plane_id];
1699         }
1700
1701         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1702         fifo_left -= sprite0_fifo_extra;
1703
1704         fifo_state->plane[PLANE_CURSOR] = 63;
1705
1706         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1707
1708         /* spread the remainder evenly */
1709         for_each_plane_id_on_crtc(crtc, plane_id) {
1710                 int plane_extra;
1711
1712                 if (fifo_left == 0)
1713                         break;
1714
1715                 if ((active_planes & BIT(plane_id)) == 0)
1716                         continue;
1717
1718                 plane_extra = min(fifo_extra, fifo_left);
1719                 fifo_state->plane[plane_id] += plane_extra;
1720                 fifo_left -= plane_extra;
1721         }
1722
1723         WARN_ON(active_planes != 0 && fifo_left != 0);
1724
1725         /* give it all to the first plane if none are active */
1726         if (active_planes == 0) {
1727                 WARN_ON(fifo_left != fifo_size);
1728                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1729         }
1730
1731         return 0;
1732 }
1733
1734 /* mark all levels starting from 'level' as invalid */
1735 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1736                                struct vlv_wm_state *wm_state, int level)
1737 {
1738         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1739
1740         for (; level < intel_wm_num_levels(dev_priv); level++) {
1741                 enum plane_id plane_id;
1742
1743                 for_each_plane_id_on_crtc(crtc, plane_id)
1744                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1745
1746                 wm_state->sr[level].cursor = USHRT_MAX;
1747                 wm_state->sr[level].plane = USHRT_MAX;
1748         }
1749 }
1750
1751 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1752 {
1753         if (wm > fifo_size)
1754                 return USHRT_MAX;
1755         else
1756                 return fifo_size - wm;
1757 }
1758
1759 /*
1760  * Starting from 'level' set all higher
1761  * levels to 'value' in the "raw" watermarks.
1762  */
1763 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1764                                  int level, enum plane_id plane_id, u16 value)
1765 {
1766         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1767         int num_levels = intel_wm_num_levels(dev_priv);
1768         bool dirty = false;
1769
1770         for (; level < num_levels; level++) {
1771                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1772
1773                 dirty |= raw->plane[plane_id] != value;
1774                 raw->plane[plane_id] = value;
1775         }
1776
1777         return dirty;
1778 }
1779
1780 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1781                                      const struct intel_plane_state *plane_state)
1782 {
1783         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1784         enum plane_id plane_id = plane->id;
1785         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1786         int level;
1787         bool dirty = false;
1788
1789         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1790                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1791                 goto out;
1792         }
1793
1794         for (level = 0; level < num_levels; level++) {
1795                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1796                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1797                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1798
1799                 if (wm > max_wm)
1800                         break;
1801
1802                 dirty |= raw->plane[plane_id] != wm;
1803                 raw->plane[plane_id] = wm;
1804         }
1805
1806         /* mark all higher levels as invalid */
1807         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1808
1809 out:
1810         if (dirty)
1811                 DRM_DEBUG_KMS("%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1812                               plane->base.name,
1813                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1814                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1815                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1816
1817         return dirty;
1818 }
1819
1820 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1821                                       enum plane_id plane_id, int level)
1822 {
1823         const struct g4x_pipe_wm *raw =
1824                 &crtc_state->wm.vlv.raw[level];
1825         const struct vlv_fifo_state *fifo_state =
1826                 &crtc_state->wm.vlv.fifo_state;
1827
1828         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1829 }
1830
1831 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1832 {
1833         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1834                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1835                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1836                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1837 }
1838
1839 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1840 {
1841         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1842         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1843         struct intel_atomic_state *state =
1844                 to_intel_atomic_state(crtc_state->base.state);
1845         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1846         const struct vlv_fifo_state *fifo_state =
1847                 &crtc_state->wm.vlv.fifo_state;
1848         int num_active_planes = hweight32(crtc_state->active_planes &
1849                                           ~BIT(PLANE_CURSOR));
1850         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->base);
1851         const struct intel_plane_state *old_plane_state;
1852         const struct intel_plane_state *new_plane_state;
1853         struct intel_plane *plane;
1854         enum plane_id plane_id;
1855         int level, ret, i;
1856         unsigned int dirty = 0;
1857
1858         for_each_oldnew_intel_plane_in_state(state, plane,
1859                                              old_plane_state,
1860                                              new_plane_state, i) {
1861                 if (new_plane_state->base.crtc != &crtc->base &&
1862                     old_plane_state->base.crtc != &crtc->base)
1863                         continue;
1864
1865                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1866                         dirty |= BIT(plane->id);
1867         }
1868
1869         /*
1870          * DSPARB registers may have been reset due to the
1871          * power well being turned off. Make sure we restore
1872          * them to a consistent state even if no primary/sprite
1873          * planes are initially active.
1874          */
1875         if (needs_modeset)
1876                 crtc_state->fifo_changed = true;
1877
1878         if (!dirty)
1879                 return 0;
1880
1881         /* cursor changes don't warrant a FIFO recompute */
1882         if (dirty & ~BIT(PLANE_CURSOR)) {
1883                 const struct intel_crtc_state *old_crtc_state =
1884                         intel_atomic_get_old_crtc_state(state, crtc);
1885                 const struct vlv_fifo_state *old_fifo_state =
1886                         &old_crtc_state->wm.vlv.fifo_state;
1887
1888                 ret = vlv_compute_fifo(crtc_state);
1889                 if (ret)
1890                         return ret;
1891
1892                 if (needs_modeset ||
1893                     memcmp(old_fifo_state, fifo_state,
1894                            sizeof(*fifo_state)) != 0)
1895                         crtc_state->fifo_changed = true;
1896         }
1897
1898         /* initially allow all levels */
1899         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1900         /*
1901          * Note that enabling cxsr with no primary/sprite planes
1902          * enabled can wedge the pipe. Hence we only allow cxsr
1903          * with exactly one enabled primary/sprite plane.
1904          */
1905         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1906
1907         for (level = 0; level < wm_state->num_levels; level++) {
1908                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1909                 const int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
1910
1911                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1912                         break;
1913
1914                 for_each_plane_id_on_crtc(crtc, plane_id) {
1915                         wm_state->wm[level].plane[plane_id] =
1916                                 vlv_invert_wm_value(raw->plane[plane_id],
1917                                                     fifo_state->plane[plane_id]);
1918                 }
1919
1920                 wm_state->sr[level].plane =
1921                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1922                                                  raw->plane[PLANE_SPRITE0],
1923                                                  raw->plane[PLANE_SPRITE1]),
1924                                             sr_fifo_size);
1925
1926                 wm_state->sr[level].cursor =
1927                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1928                                             63);
1929         }
1930
1931         if (level == 0)
1932                 return -EINVAL;
1933
1934         /* limit to only levels we can actually handle */
1935         wm_state->num_levels = level;
1936
1937         /* invalidate the higher levels */
1938         vlv_invalidate_wms(crtc, wm_state, level);
1939
1940         return 0;
1941 }
1942
1943 #define VLV_FIFO(plane, value) \
1944         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1945
1946 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1947                                    struct intel_crtc_state *crtc_state)
1948 {
1949         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1950         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1951         const struct vlv_fifo_state *fifo_state =
1952                 &crtc_state->wm.vlv.fifo_state;
1953         int sprite0_start, sprite1_start, fifo_size;
1954
1955         if (!crtc_state->fifo_changed)
1956                 return;
1957
1958         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1959         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1960         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1961
1962         WARN_ON(fifo_state->plane[PLANE_CURSOR] != 63);
1963         WARN_ON(fifo_size != 511);
1964
1965         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1966
1967         /*
1968          * uncore.lock serves a double purpose here. It allows us to
1969          * use the less expensive I915_{READ,WRITE}_FW() functions, and
1970          * it protects the DSPARB registers from getting clobbered by
1971          * parallel updates from multiple pipes.
1972          *
1973          * intel_pipe_update_start() has already disabled interrupts
1974          * for us, so a plain spin_lock() is sufficient here.
1975          */
1976         spin_lock(&dev_priv->uncore.lock);
1977
1978         switch (crtc->pipe) {
1979                 uint32_t dsparb, dsparb2, dsparb3;
1980         case PIPE_A:
1981                 dsparb = I915_READ_FW(DSPARB);
1982                 dsparb2 = I915_READ_FW(DSPARB2);
1983
1984                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1985                             VLV_FIFO(SPRITEB, 0xff));
1986                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1987                            VLV_FIFO(SPRITEB, sprite1_start));
1988
1989                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1990                              VLV_FIFO(SPRITEB_HI, 0x1));
1991                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1992                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1993
1994                 I915_WRITE_FW(DSPARB, dsparb);
1995                 I915_WRITE_FW(DSPARB2, dsparb2);
1996                 break;
1997         case PIPE_B:
1998                 dsparb = I915_READ_FW(DSPARB);
1999                 dsparb2 = I915_READ_FW(DSPARB2);
2000
2001                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2002                             VLV_FIFO(SPRITED, 0xff));
2003                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2004                            VLV_FIFO(SPRITED, sprite1_start));
2005
2006                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2007                              VLV_FIFO(SPRITED_HI, 0xff));
2008                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2009                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2010
2011                 I915_WRITE_FW(DSPARB, dsparb);
2012                 I915_WRITE_FW(DSPARB2, dsparb2);
2013                 break;
2014         case PIPE_C:
2015                 dsparb3 = I915_READ_FW(DSPARB3);
2016                 dsparb2 = I915_READ_FW(DSPARB2);
2017
2018                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2019                              VLV_FIFO(SPRITEF, 0xff));
2020                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2021                             VLV_FIFO(SPRITEF, sprite1_start));
2022
2023                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2024                              VLV_FIFO(SPRITEF_HI, 0xff));
2025                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2026                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2027
2028                 I915_WRITE_FW(DSPARB3, dsparb3);
2029                 I915_WRITE_FW(DSPARB2, dsparb2);
2030                 break;
2031         default:
2032                 break;
2033         }
2034
2035         POSTING_READ_FW(DSPARB);
2036
2037         spin_unlock(&dev_priv->uncore.lock);
2038 }
2039
2040 #undef VLV_FIFO
2041
2042 static int vlv_compute_intermediate_wm(struct drm_device *dev,
2043                                        struct intel_crtc *crtc,
2044                                        struct intel_crtc_state *new_crtc_state)
2045 {
2046         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2047         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2048         struct intel_atomic_state *intel_state =
2049                 to_intel_atomic_state(new_crtc_state->base.state);
2050         const struct intel_crtc_state *old_crtc_state =
2051                 intel_atomic_get_old_crtc_state(intel_state, crtc);
2052         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2053         int level;
2054
2055         if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
2056                 *intermediate = *optimal;
2057
2058                 intermediate->cxsr = false;
2059                 goto out;
2060         }
2061
2062         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2063         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2064                 !new_crtc_state->disable_cxsr;
2065
2066         for (level = 0; level < intermediate->num_levels; level++) {
2067                 enum plane_id plane_id;
2068
2069                 for_each_plane_id_on_crtc(crtc, plane_id) {
2070                         intermediate->wm[level].plane[plane_id] =
2071                                 min(optimal->wm[level].plane[plane_id],
2072                                     active->wm[level].plane[plane_id]);
2073                 }
2074
2075                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2076                                                     active->sr[level].plane);
2077                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2078                                                      active->sr[level].cursor);
2079         }
2080
2081         vlv_invalidate_wms(crtc, intermediate, level);
2082
2083 out:
2084         /*
2085          * If our intermediate WM are identical to the final WM, then we can
2086          * omit the post-vblank programming; only update if it's different.
2087          */
2088         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2089                 new_crtc_state->wm.need_postvbl_update = true;
2090
2091         return 0;
2092 }
2093
2094 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2095                          struct vlv_wm_values *wm)
2096 {
2097         struct intel_crtc *crtc;
2098         int num_active_crtcs = 0;
2099
2100         wm->level = dev_priv->wm.max_level;
2101         wm->cxsr = true;
2102
2103         for_each_intel_crtc(&dev_priv->drm, crtc) {
2104                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2105
2106                 if (!crtc->active)
2107                         continue;
2108
2109                 if (!wm_state->cxsr)
2110                         wm->cxsr = false;
2111
2112                 num_active_crtcs++;
2113                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2114         }
2115
2116         if (num_active_crtcs != 1)
2117                 wm->cxsr = false;
2118
2119         if (num_active_crtcs > 1)
2120                 wm->level = VLV_WM_LEVEL_PM2;
2121
2122         for_each_intel_crtc(&dev_priv->drm, crtc) {
2123                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2124                 enum pipe pipe = crtc->pipe;
2125
2126                 wm->pipe[pipe] = wm_state->wm[wm->level];
2127                 if (crtc->active && wm->cxsr)
2128                         wm->sr = wm_state->sr[wm->level];
2129
2130                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2131                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2132                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2133                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2134         }
2135 }
2136
2137 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2138 {
2139         struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2140         struct vlv_wm_values new_wm = {};
2141
2142         vlv_merge_wm(dev_priv, &new_wm);
2143
2144         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2145                 return;
2146
2147         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2148                 chv_set_memory_dvfs(dev_priv, false);
2149
2150         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2151                 chv_set_memory_pm5(dev_priv, false);
2152
2153         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2154                 _intel_set_memory_cxsr(dev_priv, false);
2155
2156         vlv_write_wm_values(dev_priv, &new_wm);
2157
2158         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2159                 _intel_set_memory_cxsr(dev_priv, true);
2160
2161         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2162                 chv_set_memory_pm5(dev_priv, true);
2163
2164         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2165                 chv_set_memory_dvfs(dev_priv, true);
2166
2167         *old_wm = new_wm;
2168 }
2169
2170 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2171                                    struct intel_crtc_state *crtc_state)
2172 {
2173         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2174         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
2175
2176         mutex_lock(&dev_priv->wm.wm_mutex);
2177         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2178         vlv_program_watermarks(dev_priv);
2179         mutex_unlock(&dev_priv->wm.wm_mutex);
2180 }
2181
2182 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2183                                     struct intel_crtc_state *crtc_state)
2184 {
2185         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2186         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
2187
2188         if (!crtc_state->wm.need_postvbl_update)
2189                 return;
2190
2191         mutex_lock(&dev_priv->wm.wm_mutex);
2192         intel_crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2193         vlv_program_watermarks(dev_priv);
2194         mutex_unlock(&dev_priv->wm.wm_mutex);
2195 }
2196
2197 static void i965_update_wm(struct intel_crtc *unused_crtc)
2198 {
2199         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2200         struct intel_crtc *crtc;
2201         int srwm = 1;
2202         int cursor_sr = 16;
2203         bool cxsr_enabled;
2204
2205         /* Calc sr entries for one plane configs */
2206         crtc = single_enabled_crtc(dev_priv);
2207         if (crtc) {
2208                 /* self-refresh has much higher latency */
2209                 static const int sr_latency_ns = 12000;
2210                 const struct drm_display_mode *adjusted_mode =
2211                         &crtc->config->base.adjusted_mode;
2212                 const struct drm_framebuffer *fb =
2213                         crtc->base.primary->state->fb;
2214                 int clock = adjusted_mode->crtc_clock;
2215                 int htotal = adjusted_mode->crtc_htotal;
2216                 int hdisplay = crtc->config->pipe_src_w;
2217                 int cpp = fb->format->cpp[0];
2218                 int entries;
2219
2220                 entries = intel_wm_method2(clock, htotal,
2221                                            hdisplay, cpp, sr_latency_ns / 100);
2222                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2223                 srwm = I965_FIFO_SIZE - entries;
2224                 if (srwm < 0)
2225                         srwm = 1;
2226                 srwm &= 0x1ff;
2227                 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
2228                               entries, srwm);
2229
2230                 entries = intel_wm_method2(clock, htotal,
2231                                            crtc->base.cursor->state->crtc_w, 4,
2232                                            sr_latency_ns / 100);
2233                 entries = DIV_ROUND_UP(entries,
2234                                        i965_cursor_wm_info.cacheline_size) +
2235                         i965_cursor_wm_info.guard_size;
2236
2237                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2238                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2239                         cursor_sr = i965_cursor_wm_info.max_wm;
2240
2241                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
2242                               "cursor %d\n", srwm, cursor_sr);
2243
2244                 cxsr_enabled = true;
2245         } else {
2246                 cxsr_enabled = false;
2247                 /* Turn off self refresh if both pipes are enabled */
2248                 intel_set_memory_cxsr(dev_priv, false);
2249         }
2250
2251         DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2252                       srwm);
2253
2254         /* 965 has limitations... */
2255         I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2256                    FW_WM(8, CURSORB) |
2257                    FW_WM(8, PLANEB) |
2258                    FW_WM(8, PLANEA));
2259         I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2260                    FW_WM(8, PLANEC_OLD));
2261         /* update cursor SR watermark */
2262         I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2263
2264         if (cxsr_enabled)
2265                 intel_set_memory_cxsr(dev_priv, true);
2266 }
2267
2268 #undef FW_WM
2269
2270 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2271 {
2272         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2273         const struct intel_watermark_params *wm_info;
2274         uint32_t fwater_lo;
2275         uint32_t fwater_hi;
2276         int cwm, srwm = 1;
2277         int fifo_size;
2278         int planea_wm, planeb_wm;
2279         struct intel_crtc *crtc, *enabled = NULL;
2280
2281         if (IS_I945GM(dev_priv))
2282                 wm_info = &i945_wm_info;
2283         else if (!IS_GEN2(dev_priv))
2284                 wm_info = &i915_wm_info;
2285         else
2286                 wm_info = &i830_a_wm_info;
2287
2288         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2289         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2290         if (intel_crtc_active(crtc)) {
2291                 const struct drm_display_mode *adjusted_mode =
2292                         &crtc->config->base.adjusted_mode;
2293                 const struct drm_framebuffer *fb =
2294                         crtc->base.primary->state->fb;
2295                 int cpp;
2296
2297                 if (IS_GEN2(dev_priv))
2298                         cpp = 4;
2299                 else
2300                         cpp = fb->format->cpp[0];
2301
2302                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2303                                                wm_info, fifo_size, cpp,
2304                                                pessimal_latency_ns);
2305                 enabled = crtc;
2306         } else {
2307                 planea_wm = fifo_size - wm_info->guard_size;
2308                 if (planea_wm > (long)wm_info->max_wm)
2309                         planea_wm = wm_info->max_wm;
2310         }
2311
2312         if (IS_GEN2(dev_priv))
2313                 wm_info = &i830_bc_wm_info;
2314
2315         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2316         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2317         if (intel_crtc_active(crtc)) {
2318                 const struct drm_display_mode *adjusted_mode =
2319                         &crtc->config->base.adjusted_mode;
2320                 const struct drm_framebuffer *fb =
2321                         crtc->base.primary->state->fb;
2322                 int cpp;
2323
2324                 if (IS_GEN2(dev_priv))
2325                         cpp = 4;
2326                 else
2327                         cpp = fb->format->cpp[0];
2328
2329                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2330                                                wm_info, fifo_size, cpp,
2331                                                pessimal_latency_ns);
2332                 if (enabled == NULL)
2333                         enabled = crtc;
2334                 else
2335                         enabled = NULL;
2336         } else {
2337                 planeb_wm = fifo_size - wm_info->guard_size;
2338                 if (planeb_wm > (long)wm_info->max_wm)
2339                         planeb_wm = wm_info->max_wm;
2340         }
2341
2342         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2343
2344         if (IS_I915GM(dev_priv) && enabled) {
2345                 struct drm_i915_gem_object *obj;
2346
2347                 obj = intel_fb_obj(enabled->base.primary->state->fb);
2348
2349                 /* self-refresh seems busted with untiled */
2350                 if (!i915_gem_object_is_tiled(obj))
2351                         enabled = NULL;
2352         }
2353
2354         /*
2355          * Overlay gets an aggressive default since video jitter is bad.
2356          */
2357         cwm = 2;
2358
2359         /* Play safe and disable self-refresh before adjusting watermarks. */
2360         intel_set_memory_cxsr(dev_priv, false);
2361
2362         /* Calc sr entries for one plane configs */
2363         if (HAS_FW_BLC(dev_priv) && enabled) {
2364                 /* self-refresh has much higher latency */
2365                 static const int sr_latency_ns = 6000;
2366                 const struct drm_display_mode *adjusted_mode =
2367                         &enabled->config->base.adjusted_mode;
2368                 const struct drm_framebuffer *fb =
2369                         enabled->base.primary->state->fb;
2370                 int clock = adjusted_mode->crtc_clock;
2371                 int htotal = adjusted_mode->crtc_htotal;
2372                 int hdisplay = enabled->config->pipe_src_w;
2373                 int cpp;
2374                 int entries;
2375
2376                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2377                         cpp = 4;
2378                 else
2379                         cpp = fb->format->cpp[0];
2380
2381                 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2382                                            sr_latency_ns / 100);
2383                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2384                 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
2385                 srwm = wm_info->fifo_size - entries;
2386                 if (srwm < 0)
2387                         srwm = 1;
2388
2389                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2390                         I915_WRITE(FW_BLC_SELF,
2391                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2392                 else
2393                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2394         }
2395
2396         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2397                       planea_wm, planeb_wm, cwm, srwm);
2398
2399         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2400         fwater_hi = (cwm & 0x1f);
2401
2402         /* Set request length to 8 cachelines per fetch */
2403         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2404         fwater_hi = fwater_hi | (1 << 8);
2405
2406         I915_WRITE(FW_BLC, fwater_lo);
2407         I915_WRITE(FW_BLC2, fwater_hi);
2408
2409         if (enabled)
2410                 intel_set_memory_cxsr(dev_priv, true);
2411 }
2412
2413 static void i845_update_wm(struct intel_crtc *unused_crtc)
2414 {
2415         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2416         struct intel_crtc *crtc;
2417         const struct drm_display_mode *adjusted_mode;
2418         uint32_t fwater_lo;
2419         int planea_wm;
2420
2421         crtc = single_enabled_crtc(dev_priv);
2422         if (crtc == NULL)
2423                 return;
2424
2425         adjusted_mode = &crtc->config->base.adjusted_mode;
2426         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2427                                        &i845_wm_info,
2428                                        dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2429                                        4, pessimal_latency_ns);
2430         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2431         fwater_lo |= (3<<8) | planea_wm;
2432
2433         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
2434
2435         I915_WRITE(FW_BLC, fwater_lo);
2436 }
2437
2438 /* latency must be in 0.1us units. */
2439 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2440                                    unsigned int cpp,
2441                                    unsigned int latency)
2442 {
2443         unsigned int ret;
2444
2445         ret = intel_wm_method1(pixel_rate, cpp, latency);
2446         ret = DIV_ROUND_UP(ret, 64) + 2;
2447
2448         return ret;
2449 }
2450
2451 /* latency must be in 0.1us units. */
2452 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2453                                    unsigned int htotal,
2454                                    unsigned int width,
2455                                    unsigned int cpp,
2456                                    unsigned int latency)
2457 {
2458         unsigned int ret;
2459
2460         ret = intel_wm_method2(pixel_rate, htotal,
2461                                width, cpp, latency);
2462         ret = DIV_ROUND_UP(ret, 64) + 2;
2463
2464         return ret;
2465 }
2466
2467 static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
2468                            uint8_t cpp)
2469 {
2470         /*
2471          * Neither of these should be possible since this function shouldn't be
2472          * called if the CRTC is off or the plane is invisible.  But let's be
2473          * extra paranoid to avoid a potential divide-by-zero if we screw up
2474          * elsewhere in the driver.
2475          */
2476         if (WARN_ON(!cpp))
2477                 return 0;
2478         if (WARN_ON(!horiz_pixels))
2479                 return 0;
2480
2481         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2482 }
2483
2484 struct ilk_wm_maximums {
2485         uint16_t pri;
2486         uint16_t spr;
2487         uint16_t cur;
2488         uint16_t fbc;
2489 };
2490
2491 /*
2492  * For both WM_PIPE and WM_LP.
2493  * mem_value must be in 0.1us units.
2494  */
2495 static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
2496                                    const struct intel_plane_state *pstate,
2497                                    uint32_t mem_value,
2498                                    bool is_lp)
2499 {
2500         uint32_t method1, method2;
2501         int cpp;
2502
2503         if (mem_value == 0)
2504                 return U32_MAX;
2505
2506         if (!intel_wm_plane_visible(cstate, pstate))
2507                 return 0;
2508
2509         cpp = pstate->base.fb->format->cpp[0];
2510
2511         method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2512
2513         if (!is_lp)
2514                 return method1;
2515
2516         method2 = ilk_wm_method2(cstate->pixel_rate,
2517                                  cstate->base.adjusted_mode.crtc_htotal,
2518                                  drm_rect_width(&pstate->base.dst),
2519                                  cpp, mem_value);
2520
2521         return min(method1, method2);
2522 }
2523
2524 /*
2525  * For both WM_PIPE and WM_LP.
2526  * mem_value must be in 0.1us units.
2527  */
2528 static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
2529                                    const struct intel_plane_state *pstate,
2530                                    uint32_t mem_value)
2531 {
2532         uint32_t method1, method2;
2533         int cpp;
2534
2535         if (mem_value == 0)
2536                 return U32_MAX;
2537
2538         if (!intel_wm_plane_visible(cstate, pstate))
2539                 return 0;
2540
2541         cpp = pstate->base.fb->format->cpp[0];
2542
2543         method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2544         method2 = ilk_wm_method2(cstate->pixel_rate,
2545                                  cstate->base.adjusted_mode.crtc_htotal,
2546                                  drm_rect_width(&pstate->base.dst),
2547                                  cpp, mem_value);
2548         return min(method1, method2);
2549 }
2550
2551 /*
2552  * For both WM_PIPE and WM_LP.
2553  * mem_value must be in 0.1us units.
2554  */
2555 static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
2556                                    const struct intel_plane_state *pstate,
2557                                    uint32_t mem_value)
2558 {
2559         int cpp;
2560
2561         if (mem_value == 0)
2562                 return U32_MAX;
2563
2564         if (!intel_wm_plane_visible(cstate, pstate))
2565                 return 0;
2566
2567         cpp = pstate->base.fb->format->cpp[0];
2568
2569         return ilk_wm_method2(cstate->pixel_rate,
2570                               cstate->base.adjusted_mode.crtc_htotal,
2571                               pstate->base.crtc_w, cpp, mem_value);
2572 }
2573
2574 /* Only for WM_LP. */
2575 static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
2576                                    const struct intel_plane_state *pstate,
2577                                    uint32_t pri_val)
2578 {
2579         int cpp;
2580
2581         if (!intel_wm_plane_visible(cstate, pstate))
2582                 return 0;
2583
2584         cpp = pstate->base.fb->format->cpp[0];
2585
2586         return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp);
2587 }
2588
2589 static unsigned int
2590 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2591 {
2592         if (INTEL_GEN(dev_priv) >= 8)
2593                 return 3072;
2594         else if (INTEL_GEN(dev_priv) >= 7)
2595                 return 768;
2596         else
2597                 return 512;
2598 }
2599
2600 static unsigned int
2601 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2602                      int level, bool is_sprite)
2603 {
2604         if (INTEL_GEN(dev_priv) >= 8)
2605                 /* BDW primary/sprite plane watermarks */
2606                 return level == 0 ? 255 : 2047;
2607         else if (INTEL_GEN(dev_priv) >= 7)
2608                 /* IVB/HSW primary/sprite plane watermarks */
2609                 return level == 0 ? 127 : 1023;
2610         else if (!is_sprite)
2611                 /* ILK/SNB primary plane watermarks */
2612                 return level == 0 ? 127 : 511;
2613         else
2614                 /* ILK/SNB sprite plane watermarks */
2615                 return level == 0 ? 63 : 255;
2616 }
2617
2618 static unsigned int
2619 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2620 {
2621         if (INTEL_GEN(dev_priv) >= 7)
2622                 return level == 0 ? 63 : 255;
2623         else
2624                 return level == 0 ? 31 : 63;
2625 }
2626
2627 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2628 {
2629         if (INTEL_GEN(dev_priv) >= 8)
2630                 return 31;
2631         else
2632                 return 15;
2633 }
2634
2635 /* Calculate the maximum primary/sprite plane watermark */
2636 static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
2637                                      int level,
2638                                      const struct intel_wm_config *config,
2639                                      enum intel_ddb_partitioning ddb_partitioning,
2640                                      bool is_sprite)
2641 {
2642         struct drm_i915_private *dev_priv = to_i915(dev);
2643         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2644
2645         /* if sprites aren't enabled, sprites get nothing */
2646         if (is_sprite && !config->sprites_enabled)
2647                 return 0;
2648
2649         /* HSW allows LP1+ watermarks even with multiple pipes */
2650         if (level == 0 || config->num_pipes_active > 1) {
2651                 fifo_size /= INTEL_INFO(dev_priv)->num_pipes;
2652
2653                 /*
2654                  * For some reason the non self refresh
2655                  * FIFO size is only half of the self
2656                  * refresh FIFO size on ILK/SNB.
2657                  */
2658                 if (INTEL_GEN(dev_priv) <= 6)
2659                         fifo_size /= 2;
2660         }
2661
2662         if (config->sprites_enabled) {
2663                 /* level 0 is always calculated with 1:1 split */
2664                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2665                         if (is_sprite)
2666                                 fifo_size *= 5;
2667                         fifo_size /= 6;
2668                 } else {
2669                         fifo_size /= 2;
2670                 }
2671         }
2672
2673         /* clamp to max that the registers can hold */
2674         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2675 }
2676
2677 /* Calculate the maximum cursor plane watermark */
2678 static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
2679                                       int level,
2680                                       const struct intel_wm_config *config)
2681 {
2682         /* HSW LP1+ watermarks w/ multiple pipes */
2683         if (level > 0 && config->num_pipes_active > 1)
2684                 return 64;
2685
2686         /* otherwise just report max that registers can hold */
2687         return ilk_cursor_wm_reg_max(to_i915(dev), level);
2688 }
2689
2690 static void ilk_compute_wm_maximums(const struct drm_device *dev,
2691                                     int level,
2692                                     const struct intel_wm_config *config,
2693                                     enum intel_ddb_partitioning ddb_partitioning,
2694                                     struct ilk_wm_maximums *max)
2695 {
2696         max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
2697         max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
2698         max->cur = ilk_cursor_wm_max(dev, level, config);
2699         max->fbc = ilk_fbc_wm_reg_max(to_i915(dev));
2700 }
2701
2702 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2703                                         int level,
2704                                         struct ilk_wm_maximums *max)
2705 {
2706         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2707         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2708         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2709         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2710 }
2711
2712 static bool ilk_validate_wm_level(int level,
2713                                   const struct ilk_wm_maximums *max,
2714                                   struct intel_wm_level *result)
2715 {
2716         bool ret;
2717
2718         /* already determined to be invalid? */
2719         if (!result->enable)
2720                 return false;
2721
2722         result->enable = result->pri_val <= max->pri &&
2723                          result->spr_val <= max->spr &&
2724                          result->cur_val <= max->cur;
2725
2726         ret = result->enable;
2727
2728         /*
2729          * HACK until we can pre-compute everything,
2730          * and thus fail gracefully if LP0 watermarks
2731          * are exceeded...
2732          */
2733         if (level == 0 && !result->enable) {
2734                 if (result->pri_val > max->pri)
2735                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2736                                       level, result->pri_val, max->pri);
2737                 if (result->spr_val > max->spr)
2738                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2739                                       level, result->spr_val, max->spr);
2740                 if (result->cur_val > max->cur)
2741                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2742                                       level, result->cur_val, max->cur);
2743
2744                 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
2745                 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
2746                 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
2747                 result->enable = true;
2748         }
2749
2750         return ret;
2751 }
2752
2753 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2754                                  const struct intel_crtc *intel_crtc,
2755                                  int level,
2756                                  struct intel_crtc_state *cstate,
2757                                  const struct intel_plane_state *pristate,
2758                                  const struct intel_plane_state *sprstate,
2759                                  const struct intel_plane_state *curstate,
2760                                  struct intel_wm_level *result)
2761 {
2762         uint16_t pri_latency = dev_priv->wm.pri_latency[level];
2763         uint16_t spr_latency = dev_priv->wm.spr_latency[level];
2764         uint16_t cur_latency = dev_priv->wm.cur_latency[level];
2765
2766         /* WM1+ latency values stored in 0.5us units */
2767         if (level > 0) {
2768                 pri_latency *= 5;
2769                 spr_latency *= 5;
2770                 cur_latency *= 5;
2771         }
2772
2773         if (pristate) {
2774                 result->pri_val = ilk_compute_pri_wm(cstate, pristate,
2775                                                      pri_latency, level);
2776                 result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val);
2777         }
2778
2779         if (sprstate)
2780                 result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency);
2781
2782         if (curstate)
2783                 result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency);
2784
2785         result->enable = true;
2786 }
2787
2788 static uint32_t
2789 hsw_compute_linetime_wm(const struct intel_crtc_state *cstate)
2790 {
2791         const struct intel_atomic_state *intel_state =
2792                 to_intel_atomic_state(cstate->base.state);
2793         const struct drm_display_mode *adjusted_mode =
2794                 &cstate->base.adjusted_mode;
2795         u32 linetime, ips_linetime;
2796
2797         if (!cstate->base.active)
2798                 return 0;
2799         if (WARN_ON(adjusted_mode->crtc_clock == 0))
2800                 return 0;
2801         if (WARN_ON(intel_state->cdclk.logical.cdclk == 0))
2802                 return 0;
2803
2804         /* The WM are computed with base on how long it takes to fill a single
2805          * row at the given clock rate, multiplied by 8.
2806          * */
2807         linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2808                                      adjusted_mode->crtc_clock);
2809         ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2810                                          intel_state->cdclk.logical.cdclk);
2811
2812         return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2813                PIPE_WM_LINETIME_TIME(linetime);
2814 }
2815
2816 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2817                                   uint16_t wm[8])
2818 {
2819         if (INTEL_GEN(dev_priv) >= 9) {
2820                 uint32_t val;
2821                 int ret, i;
2822                 int level, max_level = ilk_wm_max_level(dev_priv);
2823
2824                 /* read the first set of memory latencies[0:3] */
2825                 val = 0; /* data0 to be programmed to 0 for first set */
2826                 mutex_lock(&dev_priv->pcu_lock);
2827                 ret = sandybridge_pcode_read(dev_priv,
2828                                              GEN9_PCODE_READ_MEM_LATENCY,
2829                                              &val);
2830                 mutex_unlock(&dev_priv->pcu_lock);
2831
2832                 if (ret) {
2833                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2834                         return;
2835                 }
2836
2837                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2838                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2839                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2840                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2841                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2842                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2843                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2844
2845                 /* read the second set of memory latencies[4:7] */
2846                 val = 1; /* data0 to be programmed to 1 for second set */
2847                 mutex_lock(&dev_priv->pcu_lock);
2848                 ret = sandybridge_pcode_read(dev_priv,
2849                                              GEN9_PCODE_READ_MEM_LATENCY,
2850                                              &val);
2851                 mutex_unlock(&dev_priv->pcu_lock);
2852                 if (ret) {
2853                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2854                         return;
2855                 }
2856
2857                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2858                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2859                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2860                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2861                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2862                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2863                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2864
2865                 /*
2866                  * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2867                  * need to be disabled. We make sure to sanitize the values out
2868                  * of the punit to satisfy this requirement.
2869                  */
2870                 for (level = 1; level <= max_level; level++) {
2871                         if (wm[level] == 0) {
2872                                 for (i = level + 1; i <= max_level; i++)
2873                                         wm[i] = 0;
2874                                 break;
2875                         }
2876                 }
2877
2878                 /*
2879                  * WaWmMemoryReadLatency:skl+,glk
2880                  *
2881                  * punit doesn't take into account the read latency so we need
2882                  * to add 2us to the various latency levels we retrieve from the
2883                  * punit when level 0 response data us 0us.
2884                  */
2885                 if (wm[0] == 0) {
2886                         wm[0] += 2;
2887                         for (level = 1; level <= max_level; level++) {
2888                                 if (wm[level] == 0)
2889                                         break;
2890                                 wm[level] += 2;
2891                         }
2892                 }
2893
2894         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2895                 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2896
2897                 wm[0] = (sskpd >> 56) & 0xFF;
2898                 if (wm[0] == 0)
2899                         wm[0] = sskpd & 0xF;
2900                 wm[1] = (sskpd >> 4) & 0xFF;
2901                 wm[2] = (sskpd >> 12) & 0xFF;
2902                 wm[3] = (sskpd >> 20) & 0x1FF;
2903                 wm[4] = (sskpd >> 32) & 0x1FF;
2904         } else if (INTEL_GEN(dev_priv) >= 6) {
2905                 uint32_t sskpd = I915_READ(MCH_SSKPD);
2906
2907                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2908                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2909                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2910                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2911         } else if (INTEL_GEN(dev_priv) >= 5) {
2912                 uint32_t mltr = I915_READ(MLTR_ILK);
2913
2914                 /* ILK primary LP0 latency is 700 ns */
2915                 wm[0] = 7;
2916                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2917                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2918         } else {
2919                 MISSING_CASE(INTEL_DEVID(dev_priv));
2920         }
2921 }
2922
2923 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2924                                        uint16_t wm[5])
2925 {
2926         /* ILK sprite LP0 latency is 1300 ns */
2927         if (IS_GEN5(dev_priv))
2928                 wm[0] = 13;
2929 }
2930
2931 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2932                                        uint16_t wm[5])
2933 {
2934         /* ILK cursor LP0 latency is 1300 ns */
2935         if (IS_GEN5(dev_priv))
2936                 wm[0] = 13;
2937 }
2938
2939 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2940 {
2941         /* how many WM levels are we expecting */
2942         if (INTEL_GEN(dev_priv) >= 9)
2943                 return 7;
2944         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2945                 return 4;
2946         else if (INTEL_GEN(dev_priv) >= 6)
2947                 return 3;
2948         else
2949                 return 2;
2950 }
2951
2952 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2953                                    const char *name,
2954                                    const uint16_t wm[])
2955 {
2956         int level, max_level = ilk_wm_max_level(dev_priv);
2957
2958         for (level = 0; level <= max_level; level++) {
2959                 unsigned int latency = wm[level];
2960
2961                 if (latency == 0) {
2962                         DRM_DEBUG_KMS("%s WM%d latency not provided\n",
2963                                       name, level);
2964                         continue;
2965                 }
2966
2967                 /*
2968                  * - latencies are in us on gen9.
2969                  * - before then, WM1+ latency values are in 0.5us units
2970                  */
2971                 if (INTEL_GEN(dev_priv) >= 9)
2972                         latency *= 10;
2973                 else if (level > 0)
2974                         latency *= 5;
2975
2976                 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2977                               name, level, wm[level],
2978                               latency / 10, latency % 10);
2979         }
2980 }
2981
2982 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2983                                     uint16_t wm[5], uint16_t min)
2984 {
2985         int level, max_level = ilk_wm_max_level(dev_priv);
2986
2987         if (wm[0] >= min)
2988                 return false;
2989
2990         wm[0] = max(wm[0], min);
2991         for (level = 1; level <= max_level; level++)
2992                 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
2993
2994         return true;
2995 }
2996
2997 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2998 {
2999         bool changed;
3000
3001         /*
3002          * The BIOS provided WM memory latency values are often
3003          * inadequate for high resolution displays. Adjust them.
3004          */
3005         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12);
3006         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12);
3007         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3008
3009         if (!changed)
3010                 return;
3011
3012         DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
3013         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3014         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3015         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3016 }
3017
3018 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3019 {
3020         /*
3021          * On some SNB machines (Thinkpad X220 Tablet at least)
3022          * LP3 usage can cause vblank interrupts to be lost.
3023          * The DEIIR bit will go high but it looks like the CPU
3024          * never gets interrupted.
3025          *
3026          * It's not clear whether other interrupt source could
3027          * be affected or if this is somehow limited to vblank
3028          * interrupts only. To play it safe we disable LP3
3029          * watermarks entirely.
3030          */
3031         if (dev_priv->wm.pri_latency[3] == 0 &&
3032             dev_priv->wm.spr_latency[3] == 0 &&
3033             dev_priv->wm.cur_latency[3] == 0)
3034                 return;
3035
3036         dev_priv->wm.pri_latency[3] = 0;
3037         dev_priv->wm.spr_latency[3] = 0;
3038         dev_priv->wm.cur_latency[3] = 0;
3039
3040         DRM_DEBUG_KMS("LP3 watermarks disabled due to potential for lost interrupts\n");
3041         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3042         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3043         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3044 }
3045
3046 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3047 {
3048         intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3049
3050         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3051                sizeof(dev_priv->wm.pri_latency));
3052         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3053                sizeof(dev_priv->wm.pri_latency));
3054
3055         intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3056         intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3057
3058         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3059         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3060         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3061
3062         if (IS_GEN6(dev_priv)) {
3063                 snb_wm_latency_quirk(dev_priv);
3064                 snb_wm_lp3_irq_quirk(dev_priv);
3065         }
3066 }
3067
3068 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3069 {
3070         intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3071         intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3072 }
3073
3074 static bool ilk_validate_pipe_wm(struct drm_device *dev,
3075                                  struct intel_pipe_wm *pipe_wm)
3076 {
3077         /* LP0 watermark maximums depend on this pipe alone */
3078         const struct intel_wm_config config = {
3079                 .num_pipes_active = 1,
3080                 .sprites_enabled = pipe_wm->sprites_enabled,
3081                 .sprites_scaled = pipe_wm->sprites_scaled,
3082         };
3083         struct ilk_wm_maximums max;
3084
3085         /* LP0 watermarks always use 1/2 DDB partitioning */
3086         ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
3087
3088         /* At least LP0 must be valid */
3089         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3090                 DRM_DEBUG_KMS("LP0 watermark invalid\n");
3091                 return false;
3092         }
3093
3094         return true;
3095 }
3096
3097 /* Compute new watermarks for the pipe */
3098 static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
3099 {
3100         struct drm_atomic_state *state = cstate->base.state;
3101         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
3102         struct intel_pipe_wm *pipe_wm;
3103         struct drm_device *dev = state->dev;
3104         const struct drm_i915_private *dev_priv = to_i915(dev);
3105         struct drm_plane *plane;
3106         const struct drm_plane_state *plane_state;
3107         const struct intel_plane_state *pristate = NULL;
3108         const struct intel_plane_state *sprstate = NULL;
3109         const struct intel_plane_state *curstate = NULL;
3110         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3111         struct ilk_wm_maximums max;
3112
3113         pipe_wm = &cstate->wm.ilk.optimal;
3114
3115         drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, &cstate->base) {
3116                 const struct intel_plane_state *ps = to_intel_plane_state(plane_state);
3117
3118                 if (plane->type == DRM_PLANE_TYPE_PRIMARY)
3119                         pristate = ps;
3120                 else if (plane->type == DRM_PLANE_TYPE_OVERLAY)
3121                         sprstate = ps;
3122                 else if (plane->type == DRM_PLANE_TYPE_CURSOR)
3123                         curstate = ps;
3124         }
3125
3126         pipe_wm->pipe_enabled = cstate->base.active;
3127         if (sprstate) {
3128                 pipe_wm->sprites_enabled = sprstate->base.visible;
3129                 pipe_wm->sprites_scaled = sprstate->base.visible &&
3130                         (drm_rect_width(&sprstate->base.dst) != drm_rect_width(&sprstate->base.src) >> 16 ||
3131                          drm_rect_height(&sprstate->base.dst) != drm_rect_height(&sprstate->base.src) >> 16);
3132         }
3133
3134         usable_level = max_level;
3135
3136         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3137         if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3138                 usable_level = 1;
3139
3140         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3141         if (pipe_wm->sprites_scaled)
3142                 usable_level = 0;
3143
3144         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3145         ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate,
3146                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3147
3148         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3149                 pipe_wm->linetime = hsw_compute_linetime_wm(cstate);
3150
3151         if (!ilk_validate_pipe_wm(dev, pipe_wm))
3152                 return -EINVAL;
3153
3154         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3155
3156         for (level = 1; level <= usable_level; level++) {
3157                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3158
3159                 ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate,
3160                                      pristate, sprstate, curstate, wm);
3161
3162                 /*
3163                  * Disable any watermark level that exceeds the
3164                  * register maximums since such watermarks are
3165                  * always invalid.
3166                  */
3167                 if (!ilk_validate_wm_level(level, &max, wm)) {
3168                         memset(wm, 0, sizeof(*wm));
3169                         break;
3170                 }
3171         }
3172
3173         return 0;
3174 }
3175
3176 /*
3177  * Build a set of 'intermediate' watermark values that satisfy both the old
3178  * state and the new state.  These can be programmed to the hardware
3179  * immediately.
3180  */
3181 static int ilk_compute_intermediate_wm(struct drm_device *dev,
3182                                        struct intel_crtc *intel_crtc,
3183                                        struct intel_crtc_state *newstate)
3184 {
3185         struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3186         struct intel_atomic_state *intel_state =
3187                 to_intel_atomic_state(newstate->base.state);
3188         const struct intel_crtc_state *oldstate =
3189                 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3190         const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3191         int level, max_level = ilk_wm_max_level(to_i915(dev));
3192
3193         /*
3194          * Start with the final, target watermarks, then combine with the
3195          * currently active watermarks to get values that are safe both before
3196          * and after the vblank.
3197          */
3198         *a = newstate->wm.ilk.optimal;
3199         if (!newstate->base.active || drm_atomic_crtc_needs_modeset(&newstate->base))
3200                 return 0;
3201
3202         a->pipe_enabled |= b->pipe_enabled;
3203         a->sprites_enabled |= b->sprites_enabled;
3204         a->sprites_scaled |= b->sprites_scaled;
3205
3206         for (level = 0; level <= max_level; level++) {
3207                 struct intel_wm_level *a_wm = &a->wm[level];
3208                 const struct intel_wm_level *b_wm = &b->wm[level];
3209
3210                 a_wm->enable &= b_wm->enable;
3211                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3212                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3213                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3214                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3215         }
3216
3217         /*
3218          * We need to make sure that these merged watermark values are
3219          * actually a valid configuration themselves.  If they're not,
3220          * there's no safe way to transition from the old state to
3221          * the new state, so we need to fail the atomic transaction.
3222          */
3223         if (!ilk_validate_pipe_wm(dev, a))
3224                 return -EINVAL;
3225
3226         /*
3227          * If our intermediate WM are identical to the final WM, then we can
3228          * omit the post-vblank programming; only update if it's different.
3229          */
3230         if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3231                 newstate->wm.need_postvbl_update = true;
3232
3233         return 0;
3234 }
3235
3236 /*
3237  * Merge the watermarks from all active pipes for a specific level.
3238  */
3239 static void ilk_merge_wm_level(struct drm_device *dev,
3240                                int level,
3241                                struct intel_wm_level *ret_wm)
3242 {
3243         const struct intel_crtc *intel_crtc;
3244
3245         ret_wm->enable = true;
3246
3247         for_each_intel_crtc(dev, intel_crtc) {
3248                 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3249                 const struct intel_wm_level *wm = &active->wm[level];
3250
3251                 if (!active->pipe_enabled)
3252                         continue;
3253
3254                 /*
3255                  * The watermark values may have been used in the past,
3256                  * so we must maintain them in the registers for some
3257                  * time even if the level is now disabled.
3258                  */
3259                 if (!wm->enable)
3260                         ret_wm->enable = false;
3261
3262                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3263                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3264                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3265                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3266         }
3267 }
3268
3269 /*
3270  * Merge all low power watermarks for all active pipes.
3271  */
3272 static void ilk_wm_merge(struct drm_device *dev,
3273                          const struct intel_wm_config *config,
3274                          const struct ilk_wm_maximums *max,
3275                          struct intel_pipe_wm *merged)
3276 {
3277         struct drm_i915_private *dev_priv = to_i915(dev);
3278         int level, max_level = ilk_wm_max_level(dev_priv);
3279         int last_enabled_level = max_level;
3280
3281         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3282         if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3283             config->num_pipes_active > 1)
3284                 last_enabled_level = 0;
3285
3286         /* ILK: FBC WM must be disabled always */
3287         merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3288
3289         /* merge each WM1+ level */
3290         for (level = 1; level <= max_level; level++) {
3291                 struct intel_wm_level *wm = &merged->wm[level];
3292
3293                 ilk_merge_wm_level(dev, level, wm);
3294
3295                 if (level > last_enabled_level)
3296                         wm->enable = false;
3297                 else if (!ilk_validate_wm_level(level, max, wm))
3298                         /* make sure all following levels get disabled */
3299                         last_enabled_level = level - 1;
3300
3301                 /*
3302                  * The spec says it is preferred to disable
3303                  * FBC WMs instead of disabling a WM level.
3304                  */
3305                 if (wm->fbc_val > max->fbc) {
3306                         if (wm->enable)
3307                                 merged->fbc_wm_enabled = false;
3308                         wm->fbc_val = 0;
3309                 }
3310         }
3311
3312         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3313         /*
3314          * FIXME this is racy. FBC might get enabled later.
3315          * What we should check here is whether FBC can be
3316          * enabled sometime later.
3317          */
3318         if (IS_GEN5(dev_priv) && !merged->fbc_wm_enabled &&
3319             intel_fbc_is_active(dev_priv)) {
3320                 for (level = 2; level <= max_level; level++) {
3321                         struct intel_wm_level *wm = &merged->wm[level];
3322
3323                         wm->enable = false;
3324                 }
3325         }
3326 }
3327
3328 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3329 {
3330         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3331         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3332 }
3333
3334 /* The value we need to program into the WM_LPx latency field */
3335 static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
3336 {
3337         struct drm_i915_private *dev_priv = to_i915(dev);
3338
3339         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3340                 return 2 * level;
3341         else
3342                 return dev_priv->wm.pri_latency[level];
3343 }
3344
3345 static void ilk_compute_wm_results(struct drm_device *dev,
3346                                    const struct intel_pipe_wm *merged,
3347                                    enum intel_ddb_partitioning partitioning,
3348                                    struct ilk_wm_values *results)
3349 {
3350         struct drm_i915_private *dev_priv = to_i915(dev);
3351         struct intel_crtc *intel_crtc;
3352         int level, wm_lp;
3353
3354         results->enable_fbc_wm = merged->fbc_wm_enabled;
3355         results->partitioning = partitioning;
3356
3357         /* LP1+ register values */
3358         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3359                 const struct intel_wm_level *r;
3360
3361                 level = ilk_wm_lp_to_level(wm_lp, merged);
3362
3363                 r = &merged->wm[level];
3364
3365                 /*
3366                  * Maintain the watermark values even if the level is
3367                  * disabled. Doing otherwise could cause underruns.
3368                  */
3369                 results->wm_lp[wm_lp - 1] =
3370                         (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
3371                         (r->pri_val << WM1_LP_SR_SHIFT) |
3372                         r->cur_val;
3373
3374                 if (r->enable)
3375                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3376
3377                 if (INTEL_GEN(dev_priv) >= 8)
3378                         results->wm_lp[wm_lp - 1] |=
3379                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3380                 else
3381                         results->wm_lp[wm_lp - 1] |=
3382                                 r->fbc_val << WM1_LP_FBC_SHIFT;
3383
3384                 /*
3385                  * Always set WM1S_LP_EN when spr_val != 0, even if the
3386                  * level is disabled. Doing otherwise could cause underruns.
3387                  */
3388                 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3389                         WARN_ON(wm_lp != 1);
3390                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3391                 } else
3392                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3393         }
3394
3395         /* LP0 register values */
3396         for_each_intel_crtc(dev, intel_crtc) {
3397                 enum pipe pipe = intel_crtc->pipe;
3398                 const struct intel_wm_level *r =
3399                         &intel_crtc->wm.active.ilk.wm[0];
3400
3401                 if (WARN_ON(!r->enable))
3402                         continue;
3403
3404                 results->wm_linetime[pipe] = intel_crtc->wm.active.ilk.linetime;
3405
3406                 results->wm_pipe[pipe] =
3407                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3408                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3409                         r->cur_val;
3410         }
3411 }
3412
3413 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3414  * case both are at the same level. Prefer r1 in case they're the same. */
3415 static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
3416                                                   struct intel_pipe_wm *r1,
3417                                                   struct intel_pipe_wm *r2)
3418 {
3419         int level, max_level = ilk_wm_max_level(to_i915(dev));
3420         int level1 = 0, level2 = 0;
3421
3422         for (level = 1; level <= max_level; level++) {
3423                 if (r1->wm[level].enable)
3424                         level1 = level;
3425                 if (r2->wm[level].enable)
3426                         level2 = level;
3427         }
3428
3429         if (level1 == level2) {
3430                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3431                         return r2;
3432                 else
3433                         return r1;
3434         } else if (level1 > level2) {
3435                 return r1;
3436         } else {
3437                 return r2;
3438         }
3439 }
3440
3441 /* dirty bits used to track which watermarks need changes */
3442 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3443 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
3444 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3445 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3446 #define WM_DIRTY_FBC (1 << 24)
3447 #define WM_DIRTY_DDB (1 << 25)
3448
3449 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3450                                          const struct ilk_wm_values *old,
3451                                          const struct ilk_wm_values *new)
3452 {
3453         unsigned int dirty = 0;
3454         enum pipe pipe;
3455         int wm_lp;
3456
3457         for_each_pipe(dev_priv, pipe) {
3458                 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
3459                         dirty |= WM_DIRTY_LINETIME(pipe);
3460                         /* Must disable LP1+ watermarks too */
3461                         dirty |= WM_DIRTY_LP_ALL;
3462                 }
3463
3464                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3465                         dirty |= WM_DIRTY_PIPE(pipe);
3466                         /* Must disable LP1+ watermarks too */
3467                         dirty |= WM_DIRTY_LP_ALL;
3468                 }
3469         }
3470
3471         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3472                 dirty |= WM_DIRTY_FBC;
3473                 /* Must disable LP1+ watermarks too */
3474                 dirty |= WM_DIRTY_LP_ALL;
3475         }
3476
3477         if (old->partitioning != new->partitioning) {
3478                 dirty |= WM_DIRTY_DDB;
3479                 /* Must disable LP1+ watermarks too */
3480                 dirty |= WM_DIRTY_LP_ALL;
3481         }
3482
3483         /* LP1+ watermarks already deemed dirty, no need to continue */
3484         if (dirty & WM_DIRTY_LP_ALL)
3485                 return dirty;
3486
3487         /* Find the lowest numbered LP1+ watermark in need of an update... */
3488         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3489                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3490                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3491                         break;
3492         }
3493
3494         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3495         for (; wm_lp <= 3; wm_lp++)
3496                 dirty |= WM_DIRTY_LP(wm_lp);
3497
3498         return dirty;
3499 }
3500
3501 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3502                                unsigned int dirty)
3503 {
3504         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3505         bool changed = false;
3506
3507         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3508                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3509                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3510                 changed = true;
3511         }
3512         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3513                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3514                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3515                 changed = true;
3516         }
3517         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3518                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3519                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3520                 changed = true;
3521         }
3522
3523         /*
3524          * Don't touch WM1S_LP_EN here.
3525          * Doing so could cause underruns.
3526          */
3527
3528         return changed;
3529 }
3530
3531 /*
3532  * The spec says we shouldn't write when we don't need, because every write
3533  * causes WMs to be re-evaluated, expending some power.
3534  */
3535 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3536                                 struct ilk_wm_values *results)
3537 {
3538         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3539         unsigned int dirty;
3540         uint32_t val;
3541
3542         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3543         if (!dirty)
3544                 return;
3545
3546         _ilk_disable_lp_wm(dev_priv, dirty);
3547
3548         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3549                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3550         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3551                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3552         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3553                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3554
3555         if (dirty & WM_DIRTY_LINETIME(PIPE_A))
3556                 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
3557         if (dirty & WM_DIRTY_LINETIME(PIPE_B))
3558                 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
3559         if (dirty & WM_DIRTY_LINETIME(PIPE_C))
3560                 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
3561
3562         if (dirty & WM_DIRTY_DDB) {
3563                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3564                         val = I915_READ(WM_MISC);
3565                         if (results->partitioning == INTEL_DDB_PART_1_2)
3566                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
3567                         else
3568                                 val |= WM_MISC_DATA_PARTITION_5_6;
3569                         I915_WRITE(WM_MISC, val);
3570                 } else {
3571                         val = I915_READ(DISP_ARB_CTL2);
3572                         if (results->partitioning == INTEL_DDB_PART_1_2)
3573                                 val &= ~DISP_DATA_PARTITION_5_6;
3574                         else
3575                                 val |= DISP_DATA_PARTITION_5_6;
3576                         I915_WRITE(DISP_ARB_CTL2, val);
3577                 }
3578         }
3579
3580         if (dirty & WM_DIRTY_FBC) {
3581                 val = I915_READ(DISP_ARB_CTL);
3582                 if (results->enable_fbc_wm)
3583                         val &= ~DISP_FBC_WM_DIS;
3584                 else
3585                         val |= DISP_FBC_WM_DIS;
3586                 I915_WRITE(DISP_ARB_CTL, val);
3587         }
3588
3589         if (dirty & WM_DIRTY_LP(1) &&
3590             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3591                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3592
3593         if (INTEL_GEN(dev_priv) >= 7) {
3594                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3595                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3596                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3597                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3598         }
3599
3600         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3601                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3602         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3603                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3604         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3605                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3606
3607         dev_priv->wm.hw = *results;
3608 }
3609
3610 bool ilk_disable_lp_wm(struct drm_device *dev)
3611 {
3612         struct drm_i915_private *dev_priv = to_i915(dev);
3613
3614         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3615 }
3616
3617 static u8 intel_enabled_dbuf_slices_num(struct drm_i915_private *dev_priv)
3618 {
3619         u8 enabled_slices;
3620
3621         /* Slice 1 will always be enabled */
3622         enabled_slices = 1;
3623
3624         /* Gen prior to GEN11 have only one DBuf slice */
3625         if (INTEL_GEN(dev_priv) < 11)
3626                 return enabled_slices;
3627
3628         if (I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE)
3629                 enabled_slices++;
3630
3631         return enabled_slices;
3632 }
3633
3634 /*
3635  * FIXME: We still don't have the proper code detect if we need to apply the WA,
3636  * so assume we'll always need it in order to avoid underruns.
3637  */
3638 static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state)
3639 {
3640         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3641
3642         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
3643                 return true;
3644
3645         return false;
3646 }
3647
3648 static bool
3649 intel_has_sagv(struct drm_i915_private *dev_priv)
3650 {
3651         if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
3652             IS_CANNONLAKE(dev_priv))
3653                 return true;
3654
3655         if (IS_SKYLAKE(dev_priv) &&
3656             dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED)
3657                 return true;
3658
3659         return false;
3660 }
3661
3662 /*
3663  * SAGV dynamically adjusts the system agent voltage and clock frequencies
3664  * depending on power and performance requirements. The display engine access
3665  * to system memory is blocked during the adjustment time. Because of the
3666  * blocking time, having this enabled can cause full system hangs and/or pipe
3667  * underruns if we don't meet all of the following requirements:
3668  *
3669  *  - <= 1 pipe enabled
3670  *  - All planes can enable watermarks for latencies >= SAGV engine block time
3671  *  - We're not using an interlaced display configuration
3672  */
3673 int
3674 intel_enable_sagv(struct drm_i915_private *dev_priv)
3675 {
3676         int ret;
3677
3678         if (!intel_has_sagv(dev_priv))
3679                 return 0;
3680
3681         if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3682                 return 0;
3683
3684         DRM_DEBUG_KMS("Enabling the SAGV\n");
3685         mutex_lock(&dev_priv->pcu_lock);
3686
3687         ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3688                                       GEN9_SAGV_ENABLE);
3689
3690         /* We don't need to wait for the SAGV when enabling */
3691         mutex_unlock(&dev_priv->pcu_lock);
3692
3693         /*
3694          * Some skl systems, pre-release machines in particular,
3695          * don't actually have an SAGV.
3696          */
3697         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3698                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3699                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3700                 return 0;
3701         } else if (ret < 0) {
3702                 DRM_ERROR("Failed to enable the SAGV\n");
3703                 return ret;
3704         }
3705
3706         dev_priv->sagv_status = I915_SAGV_ENABLED;
3707         return 0;
3708 }
3709
3710 int
3711 intel_disable_sagv(struct drm_i915_private *dev_priv)
3712 {
3713         int ret;
3714
3715         if (!intel_has_sagv(dev_priv))
3716                 return 0;
3717
3718         if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3719                 return 0;
3720
3721         DRM_DEBUG_KMS("Disabling the SAGV\n");
3722         mutex_lock(&dev_priv->pcu_lock);
3723
3724         /* bspec says to keep retrying for at least 1 ms */
3725         ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3726                                 GEN9_SAGV_DISABLE,
3727                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3728                                 1);
3729         mutex_unlock(&dev_priv->pcu_lock);
3730
3731         /*
3732          * Some skl systems, pre-release machines in particular,
3733          * don't actually have an SAGV.
3734          */
3735         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3736                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3737                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3738                 return 0;
3739         } else if (ret < 0) {
3740                 DRM_ERROR("Failed to disable the SAGV (%d)\n", ret);
3741                 return ret;
3742         }
3743
3744         dev_priv->sagv_status = I915_SAGV_DISABLED;
3745         return 0;
3746 }
3747
3748 bool intel_can_enable_sagv(struct drm_atomic_state *state)
3749 {
3750         struct drm_device *dev = state->dev;
3751         struct drm_i915_private *dev_priv = to_i915(dev);
3752         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3753         struct intel_crtc *crtc;
3754         struct intel_plane *plane;
3755         struct intel_crtc_state *cstate;
3756         enum pipe pipe;
3757         int level, latency;
3758         int sagv_block_time_us;
3759
3760         if (!intel_has_sagv(dev_priv))
3761                 return false;
3762
3763         if (IS_GEN9(dev_priv))
3764                 sagv_block_time_us = 30;
3765         else if (IS_GEN10(dev_priv))
3766                 sagv_block_time_us = 20;
3767         else
3768                 sagv_block_time_us = 10;
3769
3770         /*
3771          * SKL+ workaround: bspec recommends we disable the SAGV when we have
3772          * more then one pipe enabled
3773          *
3774          * If there are no active CRTCs, no additional checks need be performed
3775          */
3776         if (hweight32(intel_state->active_crtcs) == 0)
3777                 return true;
3778         else if (hweight32(intel_state->active_crtcs) > 1)
3779                 return false;
3780
3781         /* Since we're now guaranteed to only have one active CRTC... */
3782         pipe = ffs(intel_state->active_crtcs) - 1;
3783         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
3784         cstate = to_intel_crtc_state(crtc->base.state);
3785
3786         if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3787                 return false;
3788
3789         for_each_intel_plane_on_crtc(dev, crtc, plane) {
3790                 struct skl_plane_wm *wm =
3791                         &cstate->wm.skl.optimal.planes[plane->id];
3792
3793                 /* Skip this plane if it's not enabled */
3794                 if (!wm->wm[0].plane_en)
3795                         continue;
3796
3797                 /* Find the highest enabled wm level for this plane */
3798                 for (level = ilk_wm_max_level(dev_priv);
3799                      !wm->wm[level].plane_en; --level)
3800                      { }
3801
3802                 latency = dev_priv->wm.skl_latency[level];
3803
3804                 if (skl_needs_memory_bw_wa(intel_state) &&
3805                     plane->base.state->fb->modifier ==
3806                     I915_FORMAT_MOD_X_TILED)
3807                         latency += 15;
3808
3809                 /*
3810                  * If any of the planes on this pipe don't enable wm levels that
3811                  * incur memory latencies higher than sagv_block_time_us we
3812                  * can't enable the SAGV.
3813                  */
3814                 if (latency < sagv_block_time_us)
3815                         return false;
3816         }
3817
3818         return true;
3819 }
3820
3821 static unsigned int intel_get_ddb_size(struct drm_i915_private *dev_priv,
3822                                        const struct intel_crtc_state *cstate,
3823                                        const unsigned int total_data_rate,
3824                                        const int num_active,
3825                                        struct skl_ddb_allocation *ddb)
3826 {
3827         const struct drm_display_mode *adjusted_mode;
3828         u64 total_data_bw;
3829         u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
3830
3831         WARN_ON(ddb_size == 0);
3832
3833         if (INTEL_GEN(dev_priv) < 11)
3834                 return ddb_size - 4; /* 4 blocks for bypass path allocation */
3835
3836         adjusted_mode = &cstate->base.adjusted_mode;
3837         total_data_bw = (u64)total_data_rate * drm_mode_vrefresh(adjusted_mode);
3838
3839         /*
3840          * 12GB/s is maximum BW supported by single DBuf slice.
3841          */
3842         if (total_data_bw >= GBps(12) || num_active > 1) {
3843                 ddb->enabled_slices = 2;
3844         } else {
3845                 ddb->enabled_slices = 1;
3846                 ddb_size /= 2;
3847         }
3848
3849         return ddb_size;
3850 }
3851
3852 static void
3853 skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
3854                                    const struct intel_crtc_state *cstate,
3855                                    const unsigned int total_data_rate,
3856                                    struct skl_ddb_allocation *ddb,
3857                                    struct skl_ddb_entry *alloc, /* out */
3858                                    int *num_active /* out */)
3859 {
3860         struct drm_atomic_state *state = cstate->base.state;
3861         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3862         struct drm_i915_private *dev_priv = to_i915(dev);
3863         struct drm_crtc *for_crtc = cstate->base.crtc;
3864         unsigned int pipe_size, ddb_size;
3865         int nth_active_pipe;
3866
3867         if (WARN_ON(!state) || !cstate->base.active) {
3868                 alloc->start = 0;
3869                 alloc->end = 0;
3870                 *num_active = hweight32(dev_priv->active_crtcs);
3871                 return;
3872         }
3873
3874         if (intel_state->active_pipe_changes)
3875                 *num_active = hweight32(intel_state->active_crtcs);
3876         else
3877                 *num_active = hweight32(dev_priv->active_crtcs);
3878
3879         ddb_size = intel_get_ddb_size(dev_priv, cstate, total_data_rate,
3880                                       *num_active, ddb);
3881
3882         /*
3883          * If the state doesn't change the active CRTC's, then there's
3884          * no need to recalculate; the existing pipe allocation limits
3885          * should remain unchanged.  Note that we're safe from racing
3886          * commits since any racing commit that changes the active CRTC
3887          * list would need to grab _all_ crtc locks, including the one
3888          * we currently hold.
3889          */
3890         if (!intel_state->active_pipe_changes) {
3891                 /*
3892                  * alloc may be cleared by clear_intel_crtc_state,
3893                  * copy from old state to be sure
3894                  */
3895                 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
3896                 return;
3897         }
3898
3899         nth_active_pipe = hweight32(intel_state->active_crtcs &
3900                                     (drm_crtc_mask(for_crtc) - 1));
3901         pipe_size = ddb_size / hweight32(intel_state->active_crtcs);
3902         alloc->start = nth_active_pipe * ddb_size / *num_active;
3903         alloc->end = alloc->start + pipe_size;
3904 }
3905
3906 static unsigned int skl_cursor_allocation(int num_active)
3907 {
3908         if (num_active == 1)
3909                 return 32;
3910
3911         return 8;
3912 }
3913
3914 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
3915                                        struct skl_ddb_entry *entry, u32 reg)
3916 {
3917         u16 mask;
3918
3919         if (INTEL_GEN(dev_priv) >= 11)
3920                 mask = ICL_DDB_ENTRY_MASK;
3921         else
3922                 mask = SKL_DDB_ENTRY_MASK;
3923         entry->start = reg & mask;
3924         entry->end = (reg >> DDB_ENTRY_END_SHIFT) & mask;
3925
3926         if (entry->end)
3927                 entry->end += 1;
3928 }
3929
3930 static void
3931 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
3932                            const enum pipe pipe,
3933                            const enum plane_id plane_id,
3934                            struct skl_ddb_allocation *ddb /* out */)
3935 {
3936         u32 val, val2 = 0;
3937         int fourcc, pixel_format;
3938
3939         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
3940         if (plane_id == PLANE_CURSOR) {
3941                 val = I915_READ(CUR_BUF_CFG(pipe));
3942                 skl_ddb_entry_init_from_hw(dev_priv,
3943                                            &ddb->plane[pipe][plane_id], val);
3944                 return;
3945         }
3946
3947         val = I915_READ(PLANE_CTL(pipe, plane_id));
3948
3949         /* No DDB allocated for disabled planes */
3950         if (!(val & PLANE_CTL_ENABLE))
3951                 return;
3952
3953         pixel_format = val & PLANE_CTL_FORMAT_MASK;
3954         fourcc = skl_format_to_fourcc(pixel_format,
3955                                       val & PLANE_CTL_ORDER_RGBX,
3956                                       val & PLANE_CTL_ALPHA_MASK);
3957
3958         val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
3959         val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
3960
3961         if (fourcc == DRM_FORMAT_NV12) {
3962                 skl_ddb_entry_init_from_hw(dev_priv,
3963                                            &ddb->plane[pipe][plane_id], val2);
3964                 skl_ddb_entry_init_from_hw(dev_priv,
3965                                            &ddb->uv_plane[pipe][plane_id], val);
3966         } else {
3967                 skl_ddb_entry_init_from_hw(dev_priv,
3968                                            &ddb->plane[pipe][plane_id], val);
3969         }
3970 }
3971
3972 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
3973                           struct skl_ddb_allocation *ddb /* out */)
3974 {
3975         struct intel_crtc *crtc;
3976
3977         memset(ddb, 0, sizeof(*ddb));
3978
3979         ddb->enabled_slices = intel_enabled_dbuf_slices_num(dev_priv);
3980
3981         for_each_intel_crtc(&dev_priv->drm, crtc) {
3982                 enum intel_display_power_domain power_domain;
3983                 enum plane_id plane_id;
3984                 enum pipe pipe = crtc->pipe;
3985
3986                 power_domain = POWER_DOMAIN_PIPE(pipe);
3987                 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
3988                         continue;
3989
3990                 for_each_plane_id_on_crtc(crtc, plane_id)
3991                         skl_ddb_get_hw_plane_state(dev_priv, pipe,
3992                                                    plane_id, ddb);
3993
3994                 intel_display_power_put(dev_priv, power_domain);
3995         }
3996 }
3997
3998 /*
3999  * Determines the downscale amount of a plane for the purposes of watermark calculations.
4000  * The bspec defines downscale amount as:
4001  *
4002  * """
4003  * Horizontal down scale amount = maximum[1, Horizontal source size /
4004  *                                           Horizontal destination size]
4005  * Vertical down scale amount = maximum[1, Vertical source size /
4006  *                                         Vertical destination size]
4007  * Total down scale amount = Horizontal down scale amount *
4008  *                           Vertical down scale amount
4009  * """
4010  *
4011  * Return value is provided in 16.16 fixed point form to retain fractional part.
4012  * Caller should take care of dividing & rounding off the value.
4013  */
4014 static uint_fixed_16_16_t
4015 skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
4016                            const struct intel_plane_state *pstate)
4017 {
4018         struct intel_plane *plane = to_intel_plane(pstate->base.plane);
4019         uint32_t src_w, src_h, dst_w, dst_h;
4020         uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4021         uint_fixed_16_16_t downscale_h, downscale_w;
4022
4023         if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
4024                 return u32_to_fixed16(0);
4025
4026         /* n.b., src is 16.16 fixed point, dst is whole integer */
4027         if (plane->id == PLANE_CURSOR) {
4028                 /*
4029                  * Cursors only support 0/180 degree rotation,
4030                  * hence no need to account for rotation here.
4031                  */
4032                 src_w = pstate->base.src_w >> 16;
4033                 src_h = pstate->base.src_h >> 16;
4034                 dst_w = pstate->base.crtc_w;
4035                 dst_h = pstate->base.crtc_h;
4036         } else {
4037                 /*
4038                  * Src coordinates are already rotated by 270 degrees for
4039                  * the 90/270 degree plane rotation cases (to match the
4040                  * GTT mapping), hence no need to account for rotation here.
4041                  */
4042                 src_w = drm_rect_width(&pstate->base.src) >> 16;
4043                 src_h = drm_rect_height(&pstate->base.src) >> 16;
4044                 dst_w = drm_rect_width(&pstate->base.dst);
4045                 dst_h = drm_rect_height(&pstate->base.dst);
4046         }
4047
4048         fp_w_ratio = div_fixed16(src_w, dst_w);
4049         fp_h_ratio = div_fixed16(src_h, dst_h);
4050         downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4051         downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4052
4053         return mul_fixed16(downscale_w, downscale_h);
4054 }
4055
4056 static uint_fixed_16_16_t
4057 skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
4058 {
4059         uint_fixed_16_16_t pipe_downscale = u32_to_fixed16(1);
4060
4061         if (!crtc_state->base.enable)
4062                 return pipe_downscale;
4063
4064         if (crtc_state->pch_pfit.enabled) {
4065                 uint32_t src_w, src_h, dst_w, dst_h;
4066                 uint32_t pfit_size = crtc_state->pch_pfit.size;
4067                 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4068                 uint_fixed_16_16_t downscale_h, downscale_w;
4069
4070                 src_w = crtc_state->pipe_src_w;
4071                 src_h = crtc_state->pipe_src_h;
4072                 dst_w = pfit_size >> 16;
4073                 dst_h = pfit_size & 0xffff;
4074
4075                 if (!dst_w || !dst_h)
4076                         return pipe_downscale;
4077
4078                 fp_w_ratio = div_fixed16(src_w, dst_w);
4079                 fp_h_ratio = div_fixed16(src_h, dst_h);
4080                 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4081                 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4082
4083                 pipe_downscale = mul_fixed16(downscale_w, downscale_h);
4084         }
4085
4086         return pipe_downscale;
4087 }
4088
4089 int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
4090                                   struct intel_crtc_state *cstate)
4091 {
4092         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4093         struct drm_crtc_state *crtc_state = &cstate->base;
4094         struct drm_atomic_state *state = crtc_state->state;
4095         struct drm_plane *plane;
4096         const struct drm_plane_state *pstate;
4097         struct intel_plane_state *intel_pstate;
4098         int crtc_clock, dotclk;
4099         uint32_t pipe_max_pixel_rate;
4100         uint_fixed_16_16_t pipe_downscale;
4101         uint_fixed_16_16_t max_downscale = u32_to_fixed16(1);
4102
4103         if (!cstate->base.enable)
4104                 return 0;
4105
4106         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
4107                 uint_fixed_16_16_t plane_downscale;
4108                 uint_fixed_16_16_t fp_9_div_8 = div_fixed16(9, 8);
4109                 int bpp;
4110
4111                 if (!intel_wm_plane_visible(cstate,
4112                                             to_intel_plane_state(pstate)))
4113                         continue;
4114
4115                 if (WARN_ON(!pstate->fb))
4116                         return -EINVAL;
4117
4118                 intel_pstate = to_intel_plane_state(pstate);
4119                 plane_downscale = skl_plane_downscale_amount(cstate,
4120                                                              intel_pstate);
4121                 bpp = pstate->fb->format->cpp[0] * 8;
4122                 if (bpp == 64)
4123                         plane_downscale = mul_fixed16(plane_downscale,
4124                                                       fp_9_div_8);
4125
4126                 max_downscale = max_fixed16(plane_downscale, max_downscale);
4127         }
4128         pipe_downscale = skl_pipe_downscale_amount(cstate);
4129
4130         pipe_downscale = mul_fixed16(pipe_downscale, max_downscale);
4131
4132         crtc_clock = crtc_state->adjusted_mode.crtc_clock;
4133         dotclk = to_intel_atomic_state(state)->cdclk.logical.cdclk;
4134
4135         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
4136                 dotclk *= 2;
4137
4138         pipe_max_pixel_rate = div_round_up_u32_fixed16(dotclk, pipe_downscale);
4139
4140         if (pipe_max_pixel_rate < crtc_clock) {
4141                 DRM_DEBUG_KMS("Max supported pixel clock with scaling exceeded\n");
4142                 return -EINVAL;
4143         }
4144
4145         return 0;
4146 }
4147
4148 static unsigned int
4149 skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
4150                              const struct drm_plane_state *pstate,
4151                              const int plane)
4152 {
4153         struct intel_plane *intel_plane = to_intel_plane(pstate->plane);
4154         struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
4155         uint32_t data_rate;
4156         uint32_t width = 0, height = 0;
4157         struct drm_framebuffer *fb;
4158         u32 format;
4159         uint_fixed_16_16_t down_scale_amount;
4160
4161         if (!intel_pstate->base.visible)
4162                 return 0;
4163
4164         fb = pstate->fb;
4165         format = fb->format->format;
4166
4167         if (intel_plane->id == PLANE_CURSOR)
4168                 return 0;
4169         if (plane == 1 && format != DRM_FORMAT_NV12)
4170                 return 0;
4171
4172         /*
4173          * Src coordinates are already rotated by 270 degrees for
4174          * the 90/270 degree plane rotation cases (to match the
4175          * GTT mapping), hence no need to account for rotation here.
4176          */
4177         width = drm_rect_width(&intel_pstate->base.src) >> 16;
4178         height = drm_rect_height(&intel_pstate->base.src) >> 16;
4179
4180         /* UV plane does 1/2 pixel sub-sampling */
4181         if (plane == 1 && format == DRM_FORMAT_NV12) {
4182                 width /= 2;
4183                 height /= 2;
4184         }
4185
4186         data_rate = width * height * fb->format->cpp[plane];
4187
4188         down_scale_amount = skl_plane_downscale_amount(cstate, intel_pstate);
4189
4190         return mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4191 }
4192
4193 /*
4194  * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
4195  * a 8192x4096@32bpp framebuffer:
4196  *   3 * 4096 * 8192  * 4 < 2^32
4197  */
4198 static unsigned int
4199 skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
4200                                  unsigned int *plane_data_rate,
4201                                  unsigned int *uv_plane_data_rate)
4202 {
4203         struct drm_crtc_state *cstate = &intel_cstate->base;
4204         struct drm_atomic_state *state = cstate->state;
4205         struct drm_plane *plane;
4206         const struct drm_plane_state *pstate;
4207         unsigned int total_data_rate = 0;
4208
4209         if (WARN_ON(!state))
4210                 return 0;
4211
4212         /* Calculate and cache data rate for each plane */
4213         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) {
4214                 enum plane_id plane_id = to_intel_plane(plane)->id;
4215                 unsigned int rate;
4216
4217                 /* packed/y */
4218                 rate = skl_plane_relative_data_rate(intel_cstate,
4219                                                     pstate, 0);
4220                 plane_data_rate[plane_id] = rate;
4221
4222                 total_data_rate += rate;
4223
4224                 /* uv-plane */
4225                 rate = skl_plane_relative_data_rate(intel_cstate,
4226                                                     pstate, 1);
4227                 uv_plane_data_rate[plane_id] = rate;
4228
4229                 total_data_rate += rate;
4230         }
4231
4232         return total_data_rate;
4233 }
4234
4235 static uint16_t
4236 skl_ddb_min_alloc(const struct drm_plane_state *pstate, const int plane)
4237 {
4238         struct drm_framebuffer *fb = pstate->fb;
4239         struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
4240         uint32_t src_w, src_h;
4241         uint32_t min_scanlines = 8;
4242         uint8_t plane_bpp;
4243
4244         if (WARN_ON(!fb))
4245                 return 0;
4246
4247         /* For packed formats, and uv-plane, return 0 */
4248         if (plane == 1 && fb->format->format != DRM_FORMAT_NV12)
4249                 return 0;
4250
4251         /* For Non Y-tile return 8-blocks */
4252         if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
4253             fb->modifier != I915_FORMAT_MOD_Yf_TILED &&
4254             fb->modifier != I915_FORMAT_MOD_Y_TILED_CCS &&
4255             fb->modifier != I915_FORMAT_MOD_Yf_TILED_CCS)
4256                 return 8;
4257
4258         /*
4259          * Src coordinates are already rotated by 270 degrees for
4260          * the 90/270 degree plane rotation cases (to match the
4261          * GTT mapping), hence no need to account for rotation here.
4262          */
4263         src_w = drm_rect_width(&intel_pstate->base.src) >> 16;
4264         src_h = drm_rect_height(&intel_pstate->base.src) >> 16;
4265
4266         /* Halve UV plane width and height for NV12 */
4267         if (plane == 1) {
4268                 src_w /= 2;
4269                 src_h /= 2;
4270         }
4271
4272         plane_bpp = fb->format->cpp[plane];
4273
4274         if (drm_rotation_90_or_270(pstate->rotation)) {
4275                 switch (plane_bpp) {
4276                 case 1:
4277                         min_scanlines = 32;
4278                         break;
4279                 case 2:
4280                         min_scanlines = 16;
4281                         break;
4282                 case 4:
4283                         min_scanlines = 8;
4284                         break;
4285                 case 8:
4286                         min_scanlines = 4;
4287                         break;
4288                 default:
4289                         WARN(1, "Unsupported pixel depth %u for rotation",
4290                              plane_bpp);
4291                         min_scanlines = 32;
4292                 }
4293         }
4294
4295         return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3;
4296 }
4297
4298 static void
4299 skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active,
4300                  uint16_t *minimum, uint16_t *uv_minimum)
4301 {
4302         const struct drm_plane_state *pstate;
4303         struct drm_plane *plane;
4304
4305         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, &cstate->base) {
4306                 enum plane_id plane_id = to_intel_plane(plane)->id;
4307
4308                 if (plane_id == PLANE_CURSOR)
4309                         continue;
4310
4311                 if (!pstate->visible)
4312                         continue;
4313
4314                 minimum[plane_id] = skl_ddb_min_alloc(pstate, 0);
4315                 uv_minimum[plane_id] = skl_ddb_min_alloc(pstate, 1);
4316         }
4317
4318         minimum[PLANE_CURSOR] = skl_cursor_allocation(num_active);
4319 }
4320
4321 static int
4322 skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
4323                       struct skl_ddb_allocation *ddb /* out */)
4324 {
4325         struct drm_atomic_state *state = cstate->base.state;
4326         struct drm_crtc *crtc = cstate->base.crtc;
4327         struct drm_device *dev = crtc->dev;
4328         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4329         enum pipe pipe = intel_crtc->pipe;
4330         struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb;
4331         uint16_t alloc_size, start;
4332         uint16_t minimum[I915_MAX_PLANES] = {};
4333         uint16_t uv_minimum[I915_MAX_PLANES] = {};
4334         unsigned int total_data_rate;
4335         enum plane_id plane_id;
4336         int num_active;
4337         unsigned int plane_data_rate[I915_MAX_PLANES] = {};
4338         unsigned int uv_plane_data_rate[I915_MAX_PLANES] = {};
4339         uint16_t total_min_blocks = 0;
4340
4341         /* Clear the partitioning for disabled planes. */
4342         memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
4343         memset(ddb->uv_plane[pipe], 0, sizeof(ddb->uv_plane[pipe]));
4344
4345         if (WARN_ON(!state))
4346                 return 0;
4347
4348         if (!cstate->base.active) {
4349                 alloc->start = alloc->end = 0;
4350                 return 0;
4351         }
4352
4353         total_data_rate = skl_get_total_relative_data_rate(cstate,
4354                                                            plane_data_rate,
4355                                                            uv_plane_data_rate);
4356         skl_ddb_get_pipe_allocation_limits(dev, cstate, total_data_rate, ddb,
4357                                            alloc, &num_active);
4358         alloc_size = skl_ddb_entry_size(alloc);
4359         if (alloc_size == 0)
4360                 return 0;
4361
4362         skl_ddb_calc_min(cstate, num_active, minimum, uv_minimum);
4363
4364         /*
4365          * 1. Allocate the mininum required blocks for each active plane
4366          * and allocate the cursor, it doesn't require extra allocation
4367          * proportional to the data rate.
4368          */
4369
4370         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4371                 total_min_blocks += minimum[plane_id];
4372                 total_min_blocks += uv_minimum[plane_id];
4373         }
4374
4375         if (total_min_blocks > alloc_size) {
4376                 DRM_DEBUG_KMS("Requested display configuration exceeds system DDB limitations");
4377                 DRM_DEBUG_KMS("minimum required %d/%d\n", total_min_blocks,
4378                                                         alloc_size);
4379                 return -EINVAL;
4380         }
4381
4382         alloc_size -= total_min_blocks;
4383         ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - minimum[PLANE_CURSOR];
4384         ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
4385
4386         /*
4387          * 2. Distribute the remaining space in proportion to the amount of
4388          * data each plane needs to fetch from memory.
4389          *
4390          * FIXME: we may not allocate every single block here.
4391          */
4392         if (total_data_rate == 0)
4393                 return 0;
4394
4395         start = alloc->start;
4396         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4397                 unsigned int data_rate, uv_data_rate;
4398                 uint16_t plane_blocks, uv_plane_blocks;
4399
4400                 if (plane_id == PLANE_CURSOR)
4401                         continue;
4402
4403                 data_rate = plane_data_rate[plane_id];
4404
4405                 /*
4406                  * allocation for (packed formats) or (uv-plane part of planar format):
4407                  * promote the expression to 64 bits to avoid overflowing, the
4408                  * result is < available as data_rate / total_data_rate < 1
4409                  */
4410                 plane_blocks = minimum[plane_id];
4411                 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
4412                                         total_data_rate);
4413
4414                 /* Leave disabled planes at (0,0) */
4415                 if (data_rate) {
4416                         ddb->plane[pipe][plane_id].start = start;
4417                         ddb->plane[pipe][plane_id].end = start + plane_blocks;
4418                 }
4419
4420                 start += plane_blocks;
4421
4422                 /* Allocate DDB for UV plane for planar format/NV12 */
4423                 uv_data_rate = uv_plane_data_rate[plane_id];
4424
4425                 uv_plane_blocks = uv_minimum[plane_id];
4426                 uv_plane_blocks += div_u64((uint64_t)alloc_size * uv_data_rate,
4427                                            total_data_rate);
4428
4429                 if (uv_data_rate) {
4430                         ddb->uv_plane[pipe][plane_id].start = start;
4431                         ddb->uv_plane[pipe][plane_id].end =
4432                                 start + uv_plane_blocks;
4433                 }
4434
4435                 start += uv_plane_blocks;
4436         }
4437
4438         return 0;
4439 }
4440
4441 /*
4442  * The max latency should be 257 (max the punit can code is 255 and we add 2us
4443  * for the read latency) and cpp should always be <= 8, so that
4444  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
4445  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
4446 */
4447 static uint_fixed_16_16_t
4448 skl_wm_method1(const struct drm_i915_private *dev_priv, uint32_t pixel_rate,
4449                uint8_t cpp, uint32_t latency, uint32_t dbuf_block_size)
4450 {
4451         uint32_t wm_intermediate_val;
4452         uint_fixed_16_16_t ret;
4453
4454         if (latency == 0)
4455                 return FP_16_16_MAX;
4456
4457         wm_intermediate_val = latency * pixel_rate * cpp;
4458         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
4459
4460         if (INTEL_GEN(dev_priv) >= 10)
4461                 ret = add_fixed16_u32(ret, 1);
4462
4463         return ret;
4464 }
4465
4466 static uint_fixed_16_16_t skl_wm_method2(uint32_t pixel_rate,
4467                         uint32_t pipe_htotal,
4468                         uint32_t latency,
4469                         uint_fixed_16_16_t plane_blocks_per_line)
4470 {
4471         uint32_t wm_intermediate_val;
4472         uint_fixed_16_16_t ret;
4473
4474         if (latency == 0)
4475                 return FP_16_16_MAX;
4476
4477         wm_intermediate_val = latency * pixel_rate;
4478         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
4479                                            pipe_htotal * 1000);
4480         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
4481         return ret;
4482 }
4483
4484 static uint_fixed_16_16_t
4485 intel_get_linetime_us(struct intel_crtc_state *cstate)
4486 {
4487         uint32_t pixel_rate;
4488         uint32_t crtc_htotal;
4489         uint_fixed_16_16_t linetime_us;
4490
4491         if (!cstate->base.active)
4492                 return u32_to_fixed16(0);
4493
4494         pixel_rate = cstate->pixel_rate;
4495
4496         if (WARN_ON(pixel_rate == 0))
4497                 return u32_to_fixed16(0);
4498
4499         crtc_htotal = cstate->base.adjusted_mode.crtc_htotal;
4500         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
4501
4502         return linetime_us;
4503 }
4504
4505 static uint32_t
4506 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
4507                               const struct intel_plane_state *pstate)
4508 {
4509         uint64_t adjusted_pixel_rate;
4510         uint_fixed_16_16_t downscale_amount;
4511
4512         /* Shouldn't reach here on disabled planes... */
4513         if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
4514                 return 0;
4515
4516         /*
4517          * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
4518          * with additional adjustments for plane-specific scaling.
4519          */
4520         adjusted_pixel_rate = cstate->pixel_rate;
4521         downscale_amount = skl_plane_downscale_amount(cstate, pstate);
4522
4523         return mul_round_up_u32_fixed16(adjusted_pixel_rate,
4524                                             downscale_amount);
4525 }
4526
4527 static int
4528 skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv,
4529                             struct intel_crtc_state *cstate,
4530                             const struct intel_plane_state *intel_pstate,
4531                             struct skl_wm_params *wp, int plane_id)
4532 {
4533         struct intel_plane *plane = to_intel_plane(intel_pstate->base.plane);
4534         const struct drm_plane_state *pstate = &intel_pstate->base;
4535         const struct drm_framebuffer *fb = pstate->fb;
4536         uint32_t interm_pbpl;
4537         struct intel_atomic_state *state =
4538                 to_intel_atomic_state(cstate->base.state);
4539         bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
4540
4541         if (!intel_wm_plane_visible(cstate, intel_pstate))
4542                 return 0;
4543
4544         /* only NV12 format has two planes */
4545         if (plane_id == 1 && fb->format->format != DRM_FORMAT_NV12) {
4546                 DRM_DEBUG_KMS("Non NV12 format have single plane\n");
4547                 return -EINVAL;
4548         }
4549
4550         wp->y_tiled = fb->modifier == I915_FORMAT_MOD_Y_TILED ||
4551                       fb->modifier == I915_FORMAT_MOD_Yf_TILED ||
4552                       fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4553                       fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4554         wp->x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
4555         wp->rc_surface = fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4556                          fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4557         wp->is_planar = fb->format->format == DRM_FORMAT_NV12;
4558
4559         if (plane->id == PLANE_CURSOR) {
4560                 wp->width = intel_pstate->base.crtc_w;
4561         } else {
4562                 /*
4563                  * Src coordinates are already rotated by 270 degrees for
4564                  * the 90/270 degree plane rotation cases (to match the
4565                  * GTT mapping), hence no need to account for rotation here.
4566                  */
4567                 wp->width = drm_rect_width(&intel_pstate->base.src) >> 16;
4568         }
4569
4570         if (plane_id == 1 && wp->is_planar)
4571                 wp->width /= 2;
4572
4573         wp->cpp = fb->format->cpp[plane_id];
4574         wp->plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate,
4575                                                              intel_pstate);
4576
4577         if (INTEL_GEN(dev_priv) >= 11 &&
4578             fb->modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 8)
4579                 wp->dbuf_block_size = 256;
4580         else
4581                 wp->dbuf_block_size = 512;
4582
4583         if (drm_rotation_90_or_270(pstate->rotation)) {
4584
4585                 switch (wp->cpp) {
4586                 case 1:
4587                         wp->y_min_scanlines = 16;
4588                         break;
4589                 case 2:
4590                         wp->y_min_scanlines = 8;
4591                         break;
4592                 case 4:
4593                         wp->y_min_scanlines = 4;
4594                         break;
4595                 default:
4596                         MISSING_CASE(wp->cpp);
4597                         return -EINVAL;
4598                 }
4599         } else {
4600                 wp->y_min_scanlines = 4;
4601         }
4602
4603         if (apply_memory_bw_wa)
4604                 wp->y_min_scanlines *= 2;
4605
4606         wp->plane_bytes_per_line = wp->width * wp->cpp;
4607         if (wp->y_tiled) {
4608                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
4609                                            wp->y_min_scanlines,
4610                                            wp->dbuf_block_size);
4611
4612                 if (INTEL_GEN(dev_priv) >= 10)
4613                         interm_pbpl++;
4614
4615                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
4616                                                         wp->y_min_scanlines);
4617         } else if (wp->x_tiled && IS_GEN9(dev_priv)) {
4618                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4619                                            wp->dbuf_block_size);
4620                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4621         } else {
4622                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4623                                            wp->dbuf_block_size) + 1;
4624                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4625         }
4626
4627         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
4628                                              wp->plane_blocks_per_line);
4629         wp->linetime_us = fixed16_to_u32_round_up(
4630                                         intel_get_linetime_us(cstate));
4631
4632         return 0;
4633 }
4634
4635 static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
4636                                 struct intel_crtc_state *cstate,
4637                                 const struct intel_plane_state *intel_pstate,
4638                                 uint16_t ddb_allocation,
4639                                 int level,
4640                                 const struct skl_wm_params *wp,
4641                                 const struct skl_wm_level *result_prev,
4642                                 struct skl_wm_level *result /* out */)
4643 {
4644         const struct drm_plane_state *pstate = &intel_pstate->base;
4645         uint32_t latency = dev_priv->wm.skl_latency[level];
4646         uint_fixed_16_16_t method1, method2;
4647         uint_fixed_16_16_t selected_result;
4648         uint32_t res_blocks, res_lines;
4649         struct intel_atomic_state *state =
4650                 to_intel_atomic_state(cstate->base.state);
4651         bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
4652         uint32_t min_disp_buf_needed;
4653
4654         if (latency == 0 ||
4655             !intel_wm_plane_visible(cstate, intel_pstate)) {
4656                 result->plane_en = false;
4657                 return 0;
4658         }
4659
4660         /* Display WA #1141: kbl,cfl */
4661         if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
4662             IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0)) &&
4663             dev_priv->ipc_enabled)
4664                 latency += 4;
4665
4666         if (apply_memory_bw_wa && wp->x_tiled)
4667                 latency += 15;
4668
4669         method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
4670                                  wp->cpp, latency, wp->dbuf_block_size);
4671         method2 = skl_wm_method2(wp->plane_pixel_rate,
4672                                  cstate->base.adjusted_mode.crtc_htotal,
4673                                  latency,
4674                                  wp->plane_blocks_per_line);
4675
4676         if (wp->y_tiled) {
4677                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
4678         } else {
4679                 if ((wp->cpp * cstate->base.adjusted_mode.crtc_htotal /
4680                      wp->dbuf_block_size < 1) &&
4681                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1))
4682                         selected_result = method2;
4683                 else if (ddb_allocation >=
4684                          fixed16_to_u32_round_up(wp->plane_blocks_per_line))
4685                         selected_result = min_fixed16(method1, method2);
4686                 else if (latency >= wp->linetime_us)
4687                         selected_result = min_fixed16(method1, method2);
4688                 else
4689                         selected_result = method1;
4690         }
4691
4692         res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
4693         res_lines = div_round_up_fixed16(selected_result,
4694                                          wp->plane_blocks_per_line);
4695
4696         /* Display WA #1125: skl,bxt,kbl,glk */
4697         if (level == 0 && wp->rc_surface)
4698                 res_blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
4699
4700         /* Display WA #1126: skl,bxt,kbl,glk */
4701         if (level >= 1 && level <= 7) {
4702                 if (wp->y_tiled) {
4703                         res_blocks += fixed16_to_u32_round_up(
4704                                                         wp->y_tile_minimum);
4705                         res_lines += wp->y_min_scanlines;
4706                 } else {
4707                         res_blocks++;
4708                 }
4709
4710                 /*
4711                  * Make sure result blocks for higher latency levels are atleast
4712                  * as high as level below the current level.
4713                  * Assumption in DDB algorithm optimization for special cases.
4714                  * Also covers Display WA #1125 for RC.
4715                  */
4716                 if (result_prev->plane_res_b > res_blocks)
4717                         res_blocks = result_prev->plane_res_b;
4718         }
4719
4720         if (INTEL_GEN(dev_priv) >= 11) {
4721                 if (wp->y_tiled) {
4722                         uint32_t extra_lines;
4723                         uint_fixed_16_16_t fp_min_disp_buf_needed;
4724
4725                         if (res_lines % wp->y_min_scanlines == 0)
4726                                 extra_lines = wp->y_min_scanlines;
4727                         else
4728                                 extra_lines = wp->y_min_scanlines * 2 -
4729                                               res_lines % wp->y_min_scanlines;
4730
4731                         fp_min_disp_buf_needed = mul_u32_fixed16(res_lines +
4732                                                 extra_lines,
4733                                                 wp->plane_blocks_per_line);
4734                         min_disp_buf_needed = fixed16_to_u32_round_up(
4735                                                 fp_min_disp_buf_needed);
4736                 } else {
4737                         min_disp_buf_needed = DIV_ROUND_UP(res_blocks * 11, 10);
4738                 }
4739         } else {
4740                 min_disp_buf_needed = res_blocks;
4741         }
4742
4743         if ((level > 0 && res_lines > 31) ||
4744             res_blocks >= ddb_allocation ||
4745             min_disp_buf_needed >= ddb_allocation) {
4746                 result->plane_en = false;
4747
4748                 /*
4749                  * If there are no valid level 0 watermarks, then we can't
4750                  * support this display configuration.
4751                  */
4752                 if (level) {
4753                         return 0;
4754                 } else {
4755                         struct drm_plane *plane = pstate->plane;
4756
4757                         DRM_DEBUG_KMS("Requested display configuration exceeds system watermark limitations\n");
4758                         DRM_DEBUG_KMS("[PLANE:%d:%s] blocks required = %u/%u, lines required = %u/31\n",
4759                                       plane->base.id, plane->name,
4760                                       res_blocks, ddb_allocation, res_lines);
4761                         return -EINVAL;
4762                 }
4763         }
4764
4765         /*
4766          * Display WA #826 (SKL:ALL, BXT:ALL) & #1059 (CNL:A)
4767          * disable wm level 1-7 on NV12 planes
4768          */
4769         if (wp->is_planar && level >= 1 &&
4770             (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv) ||
4771              IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))) {
4772                 result->plane_en = false;
4773                 return 0;
4774         }
4775
4776         /* The number of lines are ignored for the level 0 watermark. */
4777         result->plane_res_b = res_blocks;
4778         result->plane_res_l = res_lines;
4779         result->plane_en = true;
4780
4781         return 0;
4782 }
4783
4784 static int
4785 skl_compute_wm_levels(const struct drm_i915_private *dev_priv,
4786                       struct skl_ddb_allocation *ddb,
4787                       struct intel_crtc_state *cstate,
4788                       const struct intel_plane_state *intel_pstate,
4789                       const struct skl_wm_params *wm_params,
4790                       struct skl_plane_wm *wm,
4791                       int plane_id)
4792 {
4793         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
4794         struct drm_plane *plane = intel_pstate->base.plane;
4795         struct intel_plane *intel_plane = to_intel_plane(plane);
4796         uint16_t ddb_blocks;
4797         enum pipe pipe = intel_crtc->pipe;
4798         int level, max_level = ilk_wm_max_level(dev_priv);
4799         enum plane_id intel_plane_id = intel_plane->id;
4800         int ret;
4801
4802         if (WARN_ON(!intel_pstate->base.fb))
4803                 return -EINVAL;
4804
4805         ddb_blocks = plane_id ?
4806                      skl_ddb_entry_size(&ddb->uv_plane[pipe][intel_plane_id]) :
4807                      skl_ddb_entry_size(&ddb->plane[pipe][intel_plane_id]);
4808
4809         for (level = 0; level <= max_level; level++) {
4810                 struct skl_wm_level *result = plane_id ? &wm->uv_wm[level] :
4811                                                           &wm->wm[level];
4812                 struct skl_wm_level *result_prev;
4813
4814                 if (level)
4815                         result_prev = plane_id ? &wm->uv_wm[level - 1] :
4816                                                   &wm->wm[level - 1];
4817                 else
4818                         result_prev = plane_id ? &wm->uv_wm[0] : &wm->wm[0];
4819
4820                 ret = skl_compute_plane_wm(dev_priv,
4821                                            cstate,
4822                                            intel_pstate,
4823                                            ddb_blocks,
4824                                            level,
4825                                            wm_params,
4826                                            result_prev,
4827                                            result);
4828                 if (ret)
4829                         return ret;
4830         }
4831
4832         if (intel_pstate->base.fb->format->format == DRM_FORMAT_NV12)
4833                 wm->is_planar = true;
4834
4835         return 0;
4836 }
4837
4838 static uint32_t
4839 skl_compute_linetime_wm(struct intel_crtc_state *cstate)
4840 {
4841         struct drm_atomic_state *state = cstate->base.state;
4842         struct drm_i915_private *dev_priv = to_i915(state->dev);
4843         uint_fixed_16_16_t linetime_us;
4844         uint32_t linetime_wm;
4845
4846         linetime_us = intel_get_linetime_us(cstate);
4847
4848         if (is_fixed16_zero(linetime_us))
4849                 return 0;
4850
4851         linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us));
4852
4853         /* Display WA #1135: bxt:ALL GLK:ALL */
4854         if ((IS_BROXTON(dev_priv) || IS_GEMINILAKE(dev_priv)) &&
4855             dev_priv->ipc_enabled)
4856                 linetime_wm /= 2;
4857
4858         return linetime_wm;
4859 }
4860
4861 static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
4862                                       struct skl_wm_params *wp,
4863                                       struct skl_wm_level *wm_l0,
4864                                       uint16_t ddb_allocation,
4865                                       struct skl_wm_level *trans_wm /* out */)
4866 {
4867         struct drm_device *dev = cstate->base.crtc->dev;
4868         const struct drm_i915_private *dev_priv = to_i915(dev);
4869         uint16_t trans_min, trans_y_tile_min;
4870         const uint16_t trans_amount = 10; /* This is configurable amount */
4871         uint16_t trans_offset_b, res_blocks;
4872
4873         if (!cstate->base.active)
4874                 goto exit;
4875
4876         /* Transition WM are not recommended by HW team for GEN9 */
4877         if (INTEL_GEN(dev_priv) <= 9)
4878                 goto exit;
4879
4880         /* Transition WM don't make any sense if ipc is disabled */
4881         if (!dev_priv->ipc_enabled)
4882                 goto exit;
4883
4884         trans_min = 0;
4885         if (INTEL_GEN(dev_priv) >= 10)
4886                 trans_min = 4;
4887
4888         trans_offset_b = trans_min + trans_amount;
4889
4890         if (wp->y_tiled) {
4891                 trans_y_tile_min = (uint16_t) mul_round_up_u32_fixed16(2,
4892                                                         wp->y_tile_minimum);
4893                 res_blocks = max(wm_l0->plane_res_b, trans_y_tile_min) +
4894                                 trans_offset_b;
4895         } else {
4896                 res_blocks = wm_l0->plane_res_b + trans_offset_b;
4897
4898                 /* WA BUG:1938466 add one block for non y-tile planes */
4899                 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))
4900                         res_blocks += 1;
4901
4902         }
4903
4904         res_blocks += 1;
4905
4906         if (res_blocks < ddb_allocation) {
4907                 trans_wm->plane_res_b = res_blocks;
4908                 trans_wm->plane_en = true;
4909                 return;
4910         }
4911
4912 exit:
4913         trans_wm->plane_en = false;
4914 }
4915
4916 static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
4917                              struct skl_ddb_allocation *ddb,
4918                              struct skl_pipe_wm *pipe_wm)
4919 {
4920         struct drm_device *dev = cstate->base.crtc->dev;
4921         struct drm_crtc_state *crtc_state = &cstate->base;
4922         const struct drm_i915_private *dev_priv = to_i915(dev);
4923         struct drm_plane *plane;
4924         const struct drm_plane_state *pstate;
4925         struct skl_plane_wm *wm;
4926         int ret;
4927
4928         /*
4929          * We'll only calculate watermarks for planes that are actually
4930          * enabled, so make sure all other planes are set as disabled.
4931          */
4932         memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
4933
4934         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
4935                 const struct intel_plane_state *intel_pstate =
4936                                                 to_intel_plane_state(pstate);
4937                 enum plane_id plane_id = to_intel_plane(plane)->id;
4938                 struct skl_wm_params wm_params;
4939                 enum pipe pipe = to_intel_crtc(cstate->base.crtc)->pipe;
4940                 uint16_t ddb_blocks;
4941
4942                 wm = &pipe_wm->planes[plane_id];
4943                 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][plane_id]);
4944
4945                 ret = skl_compute_plane_wm_params(dev_priv, cstate,
4946                                                   intel_pstate, &wm_params, 0);
4947                 if (ret)
4948                         return ret;
4949
4950                 ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
4951                                             intel_pstate, &wm_params, wm, 0);
4952                 if (ret)
4953                         return ret;
4954
4955                 skl_compute_transition_wm(cstate, &wm_params, &wm->wm[0],
4956                                           ddb_blocks, &wm->trans_wm);
4957
4958                 /* uv plane watermarks must also be validated for NV12/Planar */
4959                 if (wm_params.is_planar) {
4960                         memset(&wm_params, 0, sizeof(struct skl_wm_params));
4961                         wm->is_planar = true;
4962
4963                         ret = skl_compute_plane_wm_params(dev_priv, cstate,
4964                                                           intel_pstate,
4965                                                           &wm_params, 1);
4966                         if (ret)
4967                                 return ret;
4968
4969                         ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
4970                                                     intel_pstate, &wm_params,
4971                                                     wm, 1);
4972                         if (ret)
4973                                 return ret;
4974                 }
4975         }
4976
4977         pipe_wm->linetime = skl_compute_linetime_wm(cstate);
4978
4979         return 0;
4980 }
4981
4982 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
4983                                 i915_reg_t reg,
4984                                 const struct skl_ddb_entry *entry)
4985 {
4986         if (entry->end)
4987                 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
4988         else
4989                 I915_WRITE(reg, 0);
4990 }
4991
4992 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
4993                                i915_reg_t reg,
4994                                const struct skl_wm_level *level)
4995 {
4996         uint32_t val = 0;
4997
4998         if (level->plane_en) {
4999                 val |= PLANE_WM_EN;
5000                 val |= level->plane_res_b;
5001                 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5002         }
5003
5004         I915_WRITE(reg, val);
5005 }
5006
5007 static void skl_write_plane_wm(struct intel_crtc *intel_crtc,
5008                                const struct skl_plane_wm *wm,
5009                                const struct skl_ddb_allocation *ddb,
5010                                enum plane_id plane_id)
5011 {
5012         struct drm_crtc *crtc = &intel_crtc->base;
5013         struct drm_device *dev = crtc->dev;
5014         struct drm_i915_private *dev_priv = to_i915(dev);
5015         int level, max_level = ilk_wm_max_level(dev_priv);
5016         enum pipe pipe = intel_crtc->pipe;
5017
5018         for (level = 0; level <= max_level; level++) {
5019                 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5020                                    &wm->wm[level]);
5021         }
5022         skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5023                            &wm->trans_wm);
5024
5025         skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
5026                             &ddb->plane[pipe][plane_id]);
5027         if (INTEL_GEN(dev_priv) >= 11)
5028                 return skl_ddb_entry_write(dev_priv,
5029                                            PLANE_BUF_CFG(pipe, plane_id),
5030                                            &ddb->plane[pipe][plane_id]);
5031         if (wm->is_planar) {
5032                 skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
5033                                     &ddb->uv_plane[pipe][plane_id]);
5034                 skl_ddb_entry_write(dev_priv,
5035                                     PLANE_NV12_BUF_CFG(pipe, plane_id),
5036                                     &ddb->plane[pipe][plane_id]);
5037         } else {
5038                 skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
5039                                     &ddb->plane[pipe][plane_id]);
5040                 I915_WRITE(PLANE_NV12_BUF_CFG(pipe, plane_id), 0x0);
5041         }
5042 }
5043
5044 static void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
5045                                 const struct skl_plane_wm *wm,
5046                                 const struct skl_ddb_allocation *ddb)
5047 {
5048         struct drm_crtc *crtc = &intel_crtc->base;
5049         struct drm_device *dev = crtc->dev;
5050         struct drm_i915_private *dev_priv = to_i915(dev);
5051         int level, max_level = ilk_wm_max_level(dev_priv);
5052         enum pipe pipe = intel_crtc->pipe;
5053
5054         for (level = 0; level <= max_level; level++) {
5055                 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5056                                    &wm->wm[level]);
5057         }
5058         skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5059
5060         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
5061                             &ddb->plane[pipe][PLANE_CURSOR]);
5062 }
5063
5064 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5065                          const struct skl_wm_level *l2)
5066 {
5067         if (l1->plane_en != l2->plane_en)
5068                 return false;
5069
5070         /* If both planes aren't enabled, the rest shouldn't matter */
5071         if (!l1->plane_en)
5072                 return true;
5073
5074         return (l1->plane_res_l == l2->plane_res_l &&
5075                 l1->plane_res_b == l2->plane_res_b);
5076 }
5077
5078 static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5079                                            const struct skl_ddb_entry *b)
5080 {
5081         return a->start < b->end && b->start < a->end;
5082 }
5083
5084 bool skl_ddb_allocation_overlaps(struct drm_i915_private *dev_priv,
5085                                  const struct skl_ddb_entry **entries,
5086                                  const struct skl_ddb_entry *ddb,
5087                                  int ignore)
5088 {
5089         enum pipe pipe;
5090
5091         for_each_pipe(dev_priv, pipe) {
5092                 if (pipe != ignore && entries[pipe] &&
5093                     skl_ddb_entries_overlap(ddb, entries[pipe]))
5094                         return true;
5095         }
5096
5097         return false;
5098 }
5099
5100 static int skl_update_pipe_wm(struct drm_crtc_state *cstate,
5101                               const struct skl_pipe_wm *old_pipe_wm,
5102                               struct skl_pipe_wm *pipe_wm, /* out */
5103                               struct skl_ddb_allocation *ddb, /* out */
5104                               bool *changed /* out */)
5105 {
5106         struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate);
5107         int ret;
5108
5109         ret = skl_build_pipe_wm(intel_cstate, ddb, pipe_wm);
5110         if (ret)
5111                 return ret;
5112
5113         if (!memcmp(old_pipe_wm, pipe_wm, sizeof(*pipe_wm)))
5114                 *changed = false;
5115         else
5116                 *changed = true;
5117
5118         return 0;
5119 }
5120
5121 static uint32_t
5122 pipes_modified(struct drm_atomic_state *state)
5123 {
5124         struct drm_crtc *crtc;
5125         struct drm_crtc_state *cstate;
5126         uint32_t i, ret = 0;
5127
5128         for_each_new_crtc_in_state(state, crtc, cstate, i)
5129                 ret |= drm_crtc_mask(crtc);
5130
5131         return ret;
5132 }
5133
5134 static int
5135 skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
5136 {
5137         struct drm_atomic_state *state = cstate->base.state;
5138         struct drm_device *dev = state->dev;
5139         struct drm_crtc *crtc = cstate->base.crtc;
5140         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5141         struct drm_i915_private *dev_priv = to_i915(dev);
5142         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5143         struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
5144         struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
5145         struct drm_plane_state *plane_state;
5146         struct drm_plane *plane;
5147         enum pipe pipe = intel_crtc->pipe;
5148
5149         drm_for_each_plane_mask(plane, dev, cstate->base.plane_mask) {
5150                 enum plane_id plane_id = to_intel_plane(plane)->id;
5151
5152                 if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][plane_id],
5153                                         &new_ddb->plane[pipe][plane_id]) &&
5154                     skl_ddb_entry_equal(&cur_ddb->uv_plane[pipe][plane_id],
5155                                         &new_ddb->uv_plane[pipe][plane_id]))
5156                         continue;
5157
5158                 plane_state = drm_atomic_get_plane_state(state, plane);
5159                 if (IS_ERR(plane_state))
5160                         return PTR_ERR(plane_state);
5161         }
5162
5163         return 0;
5164 }
5165
5166 static int
5167 skl_compute_ddb(struct drm_atomic_state *state)
5168 {
5169         const struct drm_i915_private *dev_priv = to_i915(state->dev);
5170         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5171         struct skl_ddb_allocation *ddb = &intel_state->wm_results.ddb;
5172         struct intel_crtc *crtc;
5173         struct intel_crtc_state *cstate;
5174         int ret, i;
5175
5176         memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb));
5177
5178         for_each_new_intel_crtc_in_state(intel_state, crtc, cstate, i) {
5179                 ret = skl_allocate_pipe_ddb(cstate, ddb);
5180                 if (ret)
5181                         return ret;
5182
5183                 ret = skl_ddb_add_affected_planes(cstate);
5184                 if (ret)
5185                         return ret;
5186         }
5187
5188         return 0;
5189 }
5190
5191 static void
5192 skl_copy_ddb_for_pipe(struct skl_ddb_values *dst,
5193                       struct skl_ddb_values *src,
5194                       enum pipe pipe)
5195 {
5196         memcpy(dst->ddb.uv_plane[pipe], src->ddb.uv_plane[pipe],
5197                sizeof(dst->ddb.uv_plane[pipe]));
5198         memcpy(dst->ddb.plane[pipe], src->ddb.plane[pipe],
5199                sizeof(dst->ddb.plane[pipe]));
5200 }
5201
5202 static void
5203 skl_print_wm_changes(const struct drm_atomic_state *state)
5204 {
5205         const struct drm_device *dev = state->dev;
5206         const struct drm_i915_private *dev_priv = to_i915(dev);
5207         const struct intel_atomic_state *intel_state =
5208                 to_intel_atomic_state(state);
5209         const struct drm_crtc *crtc;
5210         const struct drm_crtc_state *cstate;
5211         const struct intel_plane *intel_plane;
5212         const struct skl_ddb_allocation *old_ddb = &dev_priv->wm.skl_hw.ddb;
5213         const struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
5214         int i;
5215
5216         for_each_new_crtc_in_state(state, crtc, cstate, i) {
5217                 const struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5218                 enum pipe pipe = intel_crtc->pipe;
5219
5220                 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
5221                         enum plane_id plane_id = intel_plane->id;
5222                         const struct skl_ddb_entry *old, *new;
5223
5224                         old = &old_ddb->plane[pipe][plane_id];
5225                         new = &new_ddb->plane[pipe][plane_id];
5226
5227                         if (skl_ddb_entry_equal(old, new))
5228                                 continue;
5229
5230                         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n",
5231                                          intel_plane->base.base.id,
5232                                          intel_plane->base.name,
5233                                          old->start, old->end,
5234                                          new->start, new->end);
5235                 }
5236         }
5237 }
5238
5239 static int
5240 skl_ddb_add_affected_pipes(struct drm_atomic_state *state, bool *changed)
5241 {
5242         struct drm_device *dev = state->dev;
5243         const struct drm_i915_private *dev_priv = to_i915(dev);
5244         const struct drm_crtc *crtc;
5245         const struct drm_crtc_state *cstate;
5246         struct intel_crtc *intel_crtc;
5247         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5248         uint32_t realloc_pipes = pipes_modified(state);
5249         int ret, i;
5250
5251         /*
5252          * When we distrust bios wm we always need to recompute to set the
5253          * expected DDB allocations for each CRTC.
5254          */
5255         if (dev_priv->wm.distrust_bios_wm)
5256                 (*changed) = true;
5257
5258         /*
5259          * If this transaction isn't actually touching any CRTC's, don't
5260          * bother with watermark calculation.  Note that if we pass this
5261          * test, we're guaranteed to hold at least one CRTC state mutex,
5262          * which means we can safely use values like dev_priv->active_crtcs
5263          * since any racing commits that want to update them would need to
5264          * hold _all_ CRTC state mutexes.
5265          */
5266         for_each_new_crtc_in_state(state, crtc, cstate, i)
5267                 (*changed) = true;
5268
5269         if (!*changed)
5270                 return 0;
5271
5272         /*
5273          * If this is our first atomic update following hardware readout,
5274          * we can't trust the DDB that the BIOS programmed for us.  Let's
5275          * pretend that all pipes switched active status so that we'll
5276          * ensure a full DDB recompute.
5277          */
5278         if (dev_priv->wm.distrust_bios_wm) {
5279                 ret = drm_modeset_lock(&dev->mode_config.connection_mutex,
5280                                        state->acquire_ctx);
5281                 if (ret)
5282                         return ret;
5283
5284                 intel_state->active_pipe_changes = ~0;
5285
5286                 /*
5287                  * We usually only initialize intel_state->active_crtcs if we
5288                  * we're doing a modeset; make sure this field is always
5289                  * initialized during the sanitization process that happens
5290                  * on the first commit too.
5291                  */
5292                 if (!intel_state->modeset)
5293                         intel_state->active_crtcs = dev_priv->active_crtcs;
5294         }
5295
5296         /*
5297          * If the modeset changes which CRTC's are active, we need to
5298          * recompute the DDB allocation for *all* active pipes, even
5299          * those that weren't otherwise being modified in any way by this
5300          * atomic commit.  Due to the shrinking of the per-pipe allocations
5301          * when new active CRTC's are added, it's possible for a pipe that
5302          * we were already using and aren't changing at all here to suddenly
5303          * become invalid if its DDB needs exceeds its new allocation.
5304          *
5305          * Note that if we wind up doing a full DDB recompute, we can't let
5306          * any other display updates race with this transaction, so we need
5307          * to grab the lock on *all* CRTC's.
5308          */
5309         if (intel_state->active_pipe_changes) {
5310                 realloc_pipes = ~0;
5311                 intel_state->wm_results.dirty_pipes = ~0;
5312         }
5313
5314         /*
5315          * We're not recomputing for the pipes not included in the commit, so
5316          * make sure we start with the current state.
5317          */
5318         for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) {
5319                 struct intel_crtc_state *cstate;
5320
5321                 cstate = intel_atomic_get_crtc_state(state, intel_crtc);
5322                 if (IS_ERR(cstate))
5323                         return PTR_ERR(cstate);
5324         }
5325
5326         return 0;
5327 }
5328
5329 static int
5330 skl_compute_wm(struct drm_atomic_state *state)
5331 {
5332         struct drm_crtc *crtc;
5333         struct drm_crtc_state *cstate;
5334         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5335         struct skl_ddb_values *results = &intel_state->wm_results;
5336         struct skl_pipe_wm *pipe_wm;
5337         bool changed = false;
5338         int ret, i;
5339
5340         /* Clear all dirty flags */
5341         results->dirty_pipes = 0;
5342
5343         ret = skl_ddb_add_affected_pipes(state, &changed);
5344         if (ret || !changed)
5345                 return ret;
5346
5347         ret = skl_compute_ddb(state);
5348         if (ret)
5349                 return ret;
5350
5351         /*
5352          * Calculate WM's for all pipes that are part of this transaction.
5353          * Note that the DDB allocation above may have added more CRTC's that
5354          * weren't otherwise being modified (and set bits in dirty_pipes) if
5355          * pipe allocations had to change.
5356          *
5357          * FIXME:  Now that we're doing this in the atomic check phase, we
5358          * should allow skl_update_pipe_wm() to return failure in cases where
5359          * no suitable watermark values can be found.
5360          */
5361         for_each_new_crtc_in_state(state, crtc, cstate, i) {
5362                 struct intel_crtc_state *intel_cstate =
5363                         to_intel_crtc_state(cstate);
5364                 const struct skl_pipe_wm *old_pipe_wm =
5365                         &to_intel_crtc_state(crtc->state)->wm.skl.optimal;
5366
5367                 pipe_wm = &intel_cstate->wm.skl.optimal;
5368                 ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm,
5369                                          &results->ddb, &changed);
5370                 if (ret)
5371                         return ret;
5372
5373                 if (changed)
5374                         results->dirty_pipes |= drm_crtc_mask(crtc);
5375
5376                 if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
5377                         /* This pipe's WM's did not change */
5378                         continue;
5379
5380                 intel_cstate->update_wm_pre = true;
5381         }
5382
5383         skl_print_wm_changes(state);
5384
5385         return 0;
5386 }
5387
5388 static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state,
5389                                       struct intel_crtc_state *cstate)
5390 {
5391         struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc);
5392         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5393         struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
5394         const struct skl_ddb_allocation *ddb = &state->wm_results.ddb;
5395         enum pipe pipe = crtc->pipe;
5396         enum plane_id plane_id;
5397
5398         if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base)))
5399                 return;
5400
5401         I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime);
5402
5403         for_each_plane_id_on_crtc(crtc, plane_id) {
5404                 if (plane_id != PLANE_CURSOR)
5405                         skl_write_plane_wm(crtc, &pipe_wm->planes[plane_id],
5406                                            ddb, plane_id);
5407                 else
5408                         skl_write_cursor_wm(crtc, &pipe_wm->planes[plane_id],
5409                                             ddb);
5410         }
5411 }
5412
5413 static void skl_initial_wm(struct intel_atomic_state *state,
5414                            struct intel_crtc_state *cstate)
5415 {
5416         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5417         struct drm_device *dev = intel_crtc->base.dev;
5418         struct drm_i915_private *dev_priv = to_i915(dev);
5419         struct skl_ddb_values *results = &state->wm_results;
5420         struct skl_ddb_values *hw_vals = &dev_priv->wm.skl_hw;
5421         enum pipe pipe = intel_crtc->pipe;
5422
5423         if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0)
5424                 return;
5425
5426         mutex_lock(&dev_priv->wm.wm_mutex);
5427
5428         if (cstate->base.active_changed)
5429                 skl_atomic_update_crtc_wm(state, cstate);
5430
5431         skl_copy_ddb_for_pipe(hw_vals, results, pipe);
5432
5433         mutex_unlock(&dev_priv->wm.wm_mutex);
5434 }
5435
5436 static void ilk_compute_wm_config(struct drm_device *dev,
5437                                   struct intel_wm_config *config)
5438 {
5439         struct intel_crtc *crtc;
5440
5441         /* Compute the currently _active_ config */
5442         for_each_intel_crtc(dev, crtc) {
5443                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
5444
5445                 if (!wm->pipe_enabled)
5446                         continue;
5447
5448                 config->sprites_enabled |= wm->sprites_enabled;
5449                 config->sprites_scaled |= wm->sprites_scaled;
5450                 config->num_pipes_active++;
5451         }
5452 }
5453
5454 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
5455 {
5456         struct drm_device *dev = &dev_priv->drm;
5457         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
5458         struct ilk_wm_maximums max;
5459         struct intel_wm_config config = {};
5460         struct ilk_wm_values results = {};
5461         enum intel_ddb_partitioning partitioning;
5462
5463         ilk_compute_wm_config(dev, &config);
5464
5465         ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
5466         ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
5467
5468         /* 5/6 split only in single pipe config on IVB+ */
5469         if (INTEL_GEN(dev_priv) >= 7 &&
5470             config.num_pipes_active == 1 && config.sprites_enabled) {
5471                 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
5472                 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
5473
5474                 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
5475         } else {
5476                 best_lp_wm = &lp_wm_1_2;
5477         }
5478
5479         partitioning = (best_lp_wm == &lp_wm_1_2) ?
5480                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
5481
5482         ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
5483
5484         ilk_write_wm_values(dev_priv, &results);
5485 }
5486
5487 static void ilk_initial_watermarks(struct intel_atomic_state *state,
5488                                    struct intel_crtc_state *cstate)
5489 {
5490         struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5491         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5492
5493         mutex_lock(&dev_priv->wm.wm_mutex);
5494         intel_crtc->wm.active.ilk = cstate->wm.ilk.intermediate;
5495         ilk_program_watermarks(dev_priv);
5496         mutex_unlock(&dev_priv->wm.wm_mutex);
5497 }
5498
5499 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
5500                                     struct intel_crtc_state *cstate)
5501 {
5502         struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5503         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5504
5505         mutex_lock(&dev_priv->wm.wm_mutex);
5506         if (cstate->wm.need_postvbl_update) {
5507                 intel_crtc->wm.active.ilk = cstate->wm.ilk.optimal;
5508                 ilk_program_watermarks(dev_priv);
5509         }
5510         mutex_unlock(&dev_priv->wm.wm_mutex);
5511 }
5512
5513 static inline void skl_wm_level_from_reg_val(uint32_t val,
5514                                              struct skl_wm_level *level)
5515 {
5516         level->plane_en = val & PLANE_WM_EN;
5517         level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
5518         level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
5519                 PLANE_WM_LINES_MASK;
5520 }
5521
5522 void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
5523                               struct skl_pipe_wm *out)
5524 {
5525         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5526         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5527         enum pipe pipe = intel_crtc->pipe;
5528         int level, max_level;
5529         enum plane_id plane_id;
5530         uint32_t val;
5531
5532         max_level = ilk_wm_max_level(dev_priv);
5533
5534         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
5535                 struct skl_plane_wm *wm = &out->planes[plane_id];
5536
5537                 for (level = 0; level <= max_level; level++) {
5538                         if (plane_id != PLANE_CURSOR)
5539                                 val = I915_READ(PLANE_WM(pipe, plane_id, level));
5540                         else
5541                                 val = I915_READ(CUR_WM(pipe, level));
5542
5543                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
5544                 }
5545
5546                 if (plane_id != PLANE_CURSOR)
5547                         val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
5548                 else
5549                         val = I915_READ(CUR_WM_TRANS(pipe));
5550
5551                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
5552         }
5553
5554         if (!intel_crtc->active)
5555                 return;
5556
5557         out->linetime = I915_READ(PIPE_WM_LINETIME(pipe));
5558 }
5559
5560 void skl_wm_get_hw_state(struct drm_device *dev)
5561 {
5562         struct drm_i915_private *dev_priv = to_i915(dev);
5563         struct skl_ddb_values *hw = &dev_priv->wm.skl_hw;
5564         struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
5565         struct drm_crtc *crtc;
5566         struct intel_crtc *intel_crtc;
5567         struct intel_crtc_state *cstate;
5568
5569         skl_ddb_get_hw_state(dev_priv, ddb);
5570         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
5571                 intel_crtc = to_intel_crtc(crtc);
5572                 cstate = to_intel_crtc_state(crtc->state);
5573
5574                 skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal);
5575
5576                 if (intel_crtc->active)
5577                         hw->dirty_pipes |= drm_crtc_mask(crtc);
5578         }
5579
5580         if (dev_priv->active_crtcs) {
5581                 /* Fully recompute DDB on first atomic commit */
5582                 dev_priv->wm.distrust_bios_wm = true;
5583         } else {
5584                 /*
5585                  * Easy/common case; just sanitize DDB now if everything off
5586                  * Keep dbuf slice info intact
5587                  */
5588                 memset(ddb->plane, 0, sizeof(ddb->plane));
5589                 memset(ddb->uv_plane, 0, sizeof(ddb->uv_plane));
5590         }
5591 }
5592
5593 static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
5594 {
5595         struct drm_device *dev = crtc->dev;
5596         struct drm_i915_private *dev_priv = to_i915(dev);
5597         struct ilk_wm_values *hw = &dev_priv->wm.hw;
5598         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5599         struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
5600         struct intel_pipe_wm *active = &cstate->wm.ilk.optimal;
5601         enum pipe pipe = intel_crtc->pipe;
5602         static const i915_reg_t wm0_pipe_reg[] = {
5603                 [PIPE_A] = WM0_PIPEA_ILK,
5604                 [PIPE_B] = WM0_PIPEB_ILK,
5605                 [PIPE_C] = WM0_PIPEC_IVB,
5606         };
5607
5608         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
5609         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5610                 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
5611
5612         memset(active, 0, sizeof(*active));
5613
5614         active->pipe_enabled = intel_crtc->active;
5615
5616         if (active->pipe_enabled) {
5617                 u32 tmp = hw->wm_pipe[pipe];
5618
5619                 /*
5620                  * For active pipes LP0 watermark is marked as
5621                  * enabled, and LP1+ watermaks as disabled since
5622                  * we can't really reverse compute them in case
5623                  * multiple pipes are active.
5624                  */
5625                 active->wm[0].enable = true;
5626                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
5627                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
5628                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
5629                 active->linetime = hw->wm_linetime[pipe];
5630         } else {
5631                 int level, max_level = ilk_wm_max_level(dev_priv);
5632
5633                 /*
5634                  * For inactive pipes, all watermark levels
5635                  * should be marked as enabled but zeroed,
5636                  * which is what we'd compute them to.
5637                  */
5638                 for (level = 0; level <= max_level; level++)
5639                         active->wm[level].enable = true;
5640         }
5641
5642         intel_crtc->wm.active.ilk = *active;
5643 }
5644
5645 #define _FW_WM(value, plane) \
5646         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
5647 #define _FW_WM_VLV(value, plane) \
5648         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
5649
5650 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
5651                                struct g4x_wm_values *wm)
5652 {
5653         uint32_t tmp;
5654
5655         tmp = I915_READ(DSPFW1);
5656         wm->sr.plane = _FW_WM(tmp, SR);
5657         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5658         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
5659         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
5660
5661         tmp = I915_READ(DSPFW2);
5662         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
5663         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
5664         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
5665         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
5666         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5667         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
5668
5669         tmp = I915_READ(DSPFW3);
5670         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
5671         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5672         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
5673         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
5674 }
5675
5676 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
5677                                struct vlv_wm_values *wm)
5678 {
5679         enum pipe pipe;
5680         uint32_t tmp;
5681
5682         for_each_pipe(dev_priv, pipe) {
5683                 tmp = I915_READ(VLV_DDL(pipe));
5684
5685                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
5686                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5687                 wm->ddl[pipe].plane[PLANE_CURSOR] =
5688                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5689                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
5690                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5691                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
5692                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5693         }
5694
5695         tmp = I915_READ(DSPFW1);
5696         wm->sr.plane = _FW_WM(tmp, SR);
5697         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5698         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
5699         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
5700
5701         tmp = I915_READ(DSPFW2);
5702         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
5703         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5704         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
5705
5706         tmp = I915_READ(DSPFW3);
5707         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5708
5709         if (IS_CHERRYVIEW(dev_priv)) {
5710                 tmp = I915_READ(DSPFW7_CHV);
5711                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5712                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5713
5714                 tmp = I915_READ(DSPFW8_CHV);
5715                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
5716                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
5717
5718                 tmp = I915_READ(DSPFW9_CHV);
5719                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
5720                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
5721
5722                 tmp = I915_READ(DSPHOWM);
5723                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5724                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
5725                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
5726                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
5727                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5728                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5729                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5730                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5731                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5732                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5733         } else {
5734                 tmp = I915_READ(DSPFW7);
5735                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5736                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5737
5738                 tmp = I915_READ(DSPHOWM);
5739                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5740                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5741                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5742                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5743                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5744                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5745                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5746         }
5747 }
5748
5749 #undef _FW_WM
5750 #undef _FW_WM_VLV
5751
5752 void g4x_wm_get_hw_state(struct drm_device *dev)
5753 {
5754         struct drm_i915_private *dev_priv = to_i915(dev);
5755         struct g4x_wm_values *wm = &dev_priv->wm.g4x;
5756         struct intel_crtc *crtc;
5757
5758         g4x_read_wm_values(dev_priv, wm);
5759
5760         wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
5761
5762         for_each_intel_crtc(dev, crtc) {
5763                 struct intel_crtc_state *crtc_state =
5764                         to_intel_crtc_state(crtc->base.state);
5765                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
5766                 struct g4x_pipe_wm *raw;
5767                 enum pipe pipe = crtc->pipe;
5768                 enum plane_id plane_id;
5769                 int level, max_level;
5770
5771                 active->cxsr = wm->cxsr;
5772                 active->hpll_en = wm->hpll_en;
5773                 active->fbc_en = wm->fbc_en;
5774
5775                 active->sr = wm->sr;
5776                 active->hpll = wm->hpll;
5777
5778                 for_each_plane_id_on_crtc(crtc, plane_id) {
5779                         active->wm.plane[plane_id] =
5780                                 wm->pipe[pipe].plane[plane_id];
5781                 }
5782
5783                 if (wm->cxsr && wm->hpll_en)
5784                         max_level = G4X_WM_LEVEL_HPLL;
5785                 else if (wm->cxsr)
5786                         max_level = G4X_WM_LEVEL_SR;
5787                 else
5788                         max_level = G4X_WM_LEVEL_NORMAL;
5789
5790                 level = G4X_WM_LEVEL_NORMAL;
5791                 raw = &crtc_state->wm.g4x.raw[level];
5792                 for_each_plane_id_on_crtc(crtc, plane_id)
5793                         raw->plane[plane_id] = active->wm.plane[plane_id];
5794
5795                 if (++level > max_level)
5796                         goto out;
5797
5798                 raw = &crtc_state->wm.g4x.raw[level];
5799                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
5800                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
5801                 raw->plane[PLANE_SPRITE0] = 0;
5802                 raw->fbc = active->sr.fbc;
5803
5804                 if (++level > max_level)
5805                         goto out;
5806
5807                 raw = &crtc_state->wm.g4x.raw[level];
5808                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
5809                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
5810                 raw->plane[PLANE_SPRITE0] = 0;
5811                 raw->fbc = active->hpll.fbc;
5812
5813         out:
5814                 for_each_plane_id_on_crtc(crtc, plane_id)
5815                         g4x_raw_plane_wm_set(crtc_state, level,
5816                                              plane_id, USHRT_MAX);
5817                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
5818
5819                 crtc_state->wm.g4x.optimal = *active;
5820                 crtc_state->wm.g4x.intermediate = *active;
5821
5822                 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
5823                               pipe_name(pipe),
5824                               wm->pipe[pipe].plane[PLANE_PRIMARY],
5825                               wm->pipe[pipe].plane[PLANE_CURSOR],
5826                               wm->pipe[pipe].plane[PLANE_SPRITE0]);
5827         }
5828
5829         DRM_DEBUG_KMS("Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
5830                       wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
5831         DRM_DEBUG_KMS("Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
5832                       wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
5833         DRM_DEBUG_KMS("Initial SR=%s HPLL=%s FBC=%s\n",
5834                       yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
5835 }
5836
5837 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
5838 {
5839         struct intel_plane *plane;
5840         struct intel_crtc *crtc;
5841
5842         mutex_lock(&dev_priv->wm.wm_mutex);
5843
5844         for_each_intel_plane(&dev_priv->drm, plane) {
5845                 struct intel_crtc *crtc =
5846                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
5847                 struct intel_crtc_state *crtc_state =
5848                         to_intel_crtc_state(crtc->base.state);
5849                 struct intel_plane_state *plane_state =
5850                         to_intel_plane_state(plane->base.state);
5851                 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
5852                 enum plane_id plane_id = plane->id;
5853                 int level;
5854
5855                 if (plane_state->base.visible)
5856                         continue;
5857
5858                 for (level = 0; level < 3; level++) {
5859                         struct g4x_pipe_wm *raw =
5860                                 &crtc_state->wm.g4x.raw[level];
5861
5862                         raw->plane[plane_id] = 0;
5863                         wm_state->wm.plane[plane_id] = 0;
5864                 }
5865
5866                 if (plane_id == PLANE_PRIMARY) {
5867                         for (level = 0; level < 3; level++) {
5868                                 struct g4x_pipe_wm *raw =
5869                                         &crtc_state->wm.g4x.raw[level];
5870                                 raw->fbc = 0;
5871                         }
5872
5873                         wm_state->sr.fbc = 0;
5874                         wm_state->hpll.fbc = 0;
5875                         wm_state->fbc_en = false;
5876                 }
5877         }
5878
5879         for_each_intel_crtc(&dev_priv->drm, crtc) {
5880                 struct intel_crtc_state *crtc_state =
5881                         to_intel_crtc_state(crtc->base.state);
5882
5883                 crtc_state->wm.g4x.intermediate =
5884                         crtc_state->wm.g4x.optimal;
5885                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
5886         }
5887
5888         g4x_program_watermarks(dev_priv);
5889
5890         mutex_unlock(&dev_priv->wm.wm_mutex);
5891 }
5892
5893 void vlv_wm_get_hw_state(struct drm_device *dev)
5894 {
5895         struct drm_i915_private *dev_priv = to_i915(dev);
5896         struct vlv_wm_values *wm = &dev_priv->wm.vlv;
5897         struct intel_crtc *crtc;
5898         u32 val;
5899
5900         vlv_read_wm_values(dev_priv, wm);
5901
5902         wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
5903         wm->level = VLV_WM_LEVEL_PM2;
5904
5905         if (IS_CHERRYVIEW(dev_priv)) {
5906                 mutex_lock(&dev_priv->pcu_lock);
5907
5908                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5909                 if (val & DSP_MAXFIFO_PM5_ENABLE)
5910                         wm->level = VLV_WM_LEVEL_PM5;
5911
5912                 /*
5913                  * If DDR DVFS is disabled in the BIOS, Punit
5914                  * will never ack the request. So if that happens
5915                  * assume we don't have to enable/disable DDR DVFS
5916                  * dynamically. To test that just set the REQ_ACK
5917                  * bit to poke the Punit, but don't change the
5918                  * HIGH/LOW bits so that we don't actually change
5919                  * the current state.
5920                  */
5921                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
5922                 val |= FORCE_DDR_FREQ_REQ_ACK;
5923                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
5924
5925                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
5926                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
5927                         DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
5928                                       "assuming DDR DVFS is disabled\n");
5929                         dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
5930                 } else {
5931                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
5932                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
5933                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
5934                 }
5935
5936                 mutex_unlock(&dev_priv->pcu_lock);
5937         }
5938
5939         for_each_intel_crtc(dev, crtc) {
5940                 struct intel_crtc_state *crtc_state =
5941                         to_intel_crtc_state(crtc->base.state);
5942                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
5943                 const struct vlv_fifo_state *fifo_state =
5944                         &crtc_state->wm.vlv.fifo_state;
5945                 enum pipe pipe = crtc->pipe;
5946                 enum plane_id plane_id;
5947                 int level;
5948
5949                 vlv_get_fifo_size(crtc_state);
5950
5951                 active->num_levels = wm->level + 1;
5952                 active->cxsr = wm->cxsr;
5953
5954                 for (level = 0; level < active->num_levels; level++) {
5955                         struct g4x_pipe_wm *raw =
5956                                 &crtc_state->wm.vlv.raw[level];
5957
5958                         active->sr[level].plane = wm->sr.plane;
5959                         active->sr[level].cursor = wm->sr.cursor;
5960
5961                         for_each_plane_id_on_crtc(crtc, plane_id) {
5962                                 active->wm[level].plane[plane_id] =
5963                                         wm->pipe[pipe].plane[plane_id];
5964
5965                                 raw->plane[plane_id] =
5966                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
5967                                                             fifo_state->plane[plane_id]);
5968                         }
5969                 }
5970
5971                 for_each_plane_id_on_crtc(crtc, plane_id)
5972                         vlv_raw_plane_wm_set(crtc_state, level,
5973                                              plane_id, USHRT_MAX);
5974                 vlv_invalidate_wms(crtc, active, level);
5975
5976                 crtc_state->wm.vlv.optimal = *active;
5977                 crtc_state->wm.vlv.intermediate = *active;
5978
5979                 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
5980                               pipe_name(pipe),
5981                               wm->pipe[pipe].plane[PLANE_PRIMARY],
5982                               wm->pipe[pipe].plane[PLANE_CURSOR],
5983                               wm->pipe[pipe].plane[PLANE_SPRITE0],
5984                               wm->pipe[pipe].plane[PLANE_SPRITE1]);
5985         }
5986
5987         DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
5988                       wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
5989 }
5990
5991 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
5992 {
5993         struct intel_plane *plane;
5994         struct intel_crtc *crtc;
5995
5996         mutex_lock(&dev_priv->wm.wm_mutex);
5997
5998         for_each_intel_plane(&dev_priv->drm, plane) {
5999                 struct intel_crtc *crtc =
6000                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6001                 struct intel_crtc_state *crtc_state =
6002                         to_intel_crtc_state(crtc->base.state);
6003                 struct intel_plane_state *plane_state =
6004                         to_intel_plane_state(plane->base.state);
6005                 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6006                 const struct vlv_fifo_state *fifo_state =
6007                         &crtc_state->wm.vlv.fifo_state;
6008                 enum plane_id plane_id = plane->id;
6009                 int level;
6010
6011                 if (plane_state->base.visible)
6012                         continue;
6013
6014                 for (level = 0; level < wm_state->num_levels; level++) {
6015                         struct g4x_pipe_wm *raw =
6016                                 &crtc_state->wm.vlv.raw[level];
6017
6018                         raw->plane[plane_id] = 0;
6019
6020                         wm_state->wm[level].plane[plane_id] =
6021                                 vlv_invert_wm_value(raw->plane[plane_id],
6022                                                     fifo_state->plane[plane_id]);
6023                 }
6024         }
6025
6026         for_each_intel_crtc(&dev_priv->drm, crtc) {
6027                 struct intel_crtc_state *crtc_state =
6028                         to_intel_crtc_state(crtc->base.state);
6029
6030                 crtc_state->wm.vlv.intermediate =
6031                         crtc_state->wm.vlv.optimal;
6032                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6033         }
6034
6035         vlv_program_watermarks(dev_priv);
6036
6037         mutex_unlock(&dev_priv->wm.wm_mutex);
6038 }
6039
6040 /*
6041  * FIXME should probably kill this and improve
6042  * the real watermark readout/sanitation instead
6043  */
6044 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6045 {
6046         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6047         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6048         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6049
6050         /*
6051          * Don't touch WM1S_LP_EN here.
6052          * Doing so could cause underruns.
6053          */
6054 }
6055
6056 void ilk_wm_get_hw_state(struct drm_device *dev)
6057 {
6058         struct drm_i915_private *dev_priv = to_i915(dev);
6059         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6060         struct drm_crtc *crtc;
6061
6062         ilk_init_lp_watermarks(dev_priv);
6063
6064         for_each_crtc(dev, crtc)
6065                 ilk_pipe_wm_get_hw_state(crtc);
6066
6067         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6068         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6069         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6070
6071         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6072         if (INTEL_GEN(dev_priv) >= 7) {
6073                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6074                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6075         }
6076
6077         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6078                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6079                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6080         else if (IS_IVYBRIDGE(dev_priv))
6081                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6082                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6083
6084         hw->enable_fbc_wm =
6085                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6086 }
6087
6088 /**
6089  * intel_update_watermarks - update FIFO watermark values based on current modes
6090  * @crtc: the #intel_crtc on which to compute the WM
6091  *
6092  * Calculate watermark values for the various WM regs based on current mode
6093  * and plane configuration.
6094  *
6095  * There are several cases to deal with here:
6096  *   - normal (i.e. non-self-refresh)
6097  *   - self-refresh (SR) mode
6098  *   - lines are large relative to FIFO size (buffer can hold up to 2)
6099  *   - lines are small relative to FIFO size (buffer can hold more than 2
6100  *     lines), so need to account for TLB latency
6101  *
6102  *   The normal calculation is:
6103  *     watermark = dotclock * bytes per pixel * latency
6104  *   where latency is platform & configuration dependent (we assume pessimal
6105  *   values here).
6106  *
6107  *   The SR calculation is:
6108  *     watermark = (trunc(latency/line time)+1) * surface width *
6109  *       bytes per pixel
6110  *   where
6111  *     line time = htotal / dotclock
6112  *     surface width = hdisplay for normal plane and 64 for cursor
6113  *   and latency is assumed to be high, as above.
6114  *
6115  * The final value programmed to the register should always be rounded up,
6116  * and include an extra 2 entries to account for clock crossings.
6117  *
6118  * We don't use the sprite, so we can ignore that.  And on Crestline we have
6119  * to set the non-SR watermarks to 8.
6120  */
6121 void intel_update_watermarks(struct intel_crtc *crtc)
6122 {
6123         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6124
6125         if (dev_priv->display.update_wm)
6126                 dev_priv->display.update_wm(crtc);
6127 }
6128
6129 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6130 {
6131         u32 val;
6132
6133         /* Display WA #0477 WaDisableIPC: skl */
6134         if (IS_SKYLAKE(dev_priv)) {
6135                 dev_priv->ipc_enabled = false;
6136                 return;
6137         }
6138
6139         val = I915_READ(DISP_ARB_CTL2);
6140
6141         if (dev_priv->ipc_enabled)
6142                 val |= DISP_IPC_ENABLE;
6143         else
6144                 val &= ~DISP_IPC_ENABLE;
6145
6146         I915_WRITE(DISP_ARB_CTL2, val);
6147 }
6148
6149 void intel_init_ipc(struct drm_i915_private *dev_priv)
6150 {
6151         dev_priv->ipc_enabled = false;
6152         if (!HAS_IPC(dev_priv))
6153                 return;
6154
6155         dev_priv->ipc_enabled = true;
6156         intel_enable_ipc(dev_priv);
6157 }
6158
6159 /*
6160  * Lock protecting IPS related data structures
6161  */
6162 DEFINE_SPINLOCK(mchdev_lock);
6163
6164 /* Global for IPS driver to get at the current i915 device. Protected by
6165  * mchdev_lock. */
6166 static struct drm_i915_private *i915_mch_dev;
6167
6168 bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val)
6169 {
6170         u16 rgvswctl;
6171
6172         lockdep_assert_held(&mchdev_lock);
6173
6174         rgvswctl = I915_READ16(MEMSWCTL);
6175         if (rgvswctl & MEMCTL_CMD_STS) {
6176                 DRM_DEBUG("gpu busy, RCS change rejected\n");
6177                 return false; /* still busy with another command */
6178         }
6179
6180         rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
6181                 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
6182         I915_WRITE16(MEMSWCTL, rgvswctl);
6183         POSTING_READ16(MEMSWCTL);
6184
6185         rgvswctl |= MEMCTL_CMD_STS;
6186         I915_WRITE16(MEMSWCTL, rgvswctl);
6187
6188         return true;
6189 }
6190
6191 static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
6192 {
6193         u32 rgvmodectl;
6194         u8 fmax, fmin, fstart, vstart;
6195
6196         spin_lock_irq(&mchdev_lock);
6197
6198         rgvmodectl = I915_READ(MEMMODECTL);
6199
6200         /* Enable temp reporting */
6201         I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
6202         I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
6203
6204         /* 100ms RC evaluation intervals */
6205         I915_WRITE(RCUPEI, 100000);
6206         I915_WRITE(RCDNEI, 100000);
6207
6208         /* Set max/min thresholds to 90ms and 80ms respectively */
6209         I915_WRITE(RCBMAXAVG, 90000);
6210         I915_WRITE(RCBMINAVG, 80000);
6211
6212         I915_WRITE(MEMIHYST, 1);
6213
6214         /* Set up min, max, and cur for interrupt handling */
6215         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
6216         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
6217         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
6218                 MEMMODE_FSTART_SHIFT;
6219
6220         vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
6221                 PXVFREQ_PX_SHIFT;
6222
6223         dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
6224         dev_priv->ips.fstart = fstart;
6225
6226         dev_priv->ips.max_delay = fstart;
6227         dev_priv->ips.min_delay = fmin;
6228         dev_priv->ips.cur_delay = fstart;
6229
6230         DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
6231                          fmax, fmin, fstart);
6232
6233         I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
6234
6235         /*
6236          * Interrupts will be enabled in ironlake_irq_postinstall
6237          */
6238
6239         I915_WRITE(VIDSTART, vstart);
6240         POSTING_READ(VIDSTART);
6241
6242         rgvmodectl |= MEMMODE_SWMODE_EN;
6243         I915_WRITE(MEMMODECTL, rgvmodectl);
6244
6245         if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
6246                 DRM_ERROR("stuck trying to change perf mode\n");
6247         mdelay(1);
6248
6249         ironlake_set_drps(dev_priv, fstart);
6250
6251         dev_priv->ips.last_count1 = I915_READ(DMIEC) +
6252                 I915_READ(DDREC) + I915_READ(CSIEC);
6253         dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
6254         dev_priv->ips.last_count2 = I915_READ(GFXEC);
6255         dev_priv->ips.last_time2 = ktime_get_raw_ns();
6256
6257         spin_unlock_irq(&mchdev_lock);
6258 }
6259
6260 static void ironlake_disable_drps(struct drm_i915_private *dev_priv)
6261 {
6262         u16 rgvswctl;
6263
6264         spin_lock_irq(&mchdev_lock);
6265
6266         rgvswctl = I915_READ16(MEMSWCTL);
6267
6268         /* Ack interrupts, disable EFC interrupt */
6269         I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
6270         I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
6271         I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
6272         I915_WRITE(DEIIR, DE_PCU_EVENT);
6273         I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
6274
6275         /* Go back to the starting frequency */
6276         ironlake_set_drps(dev_priv, dev_priv->ips.fstart);
6277         mdelay(1);
6278         rgvswctl |= MEMCTL_CMD_STS;
6279         I915_WRITE(MEMSWCTL, rgvswctl);
6280         mdelay(1);
6281
6282         spin_unlock_irq(&mchdev_lock);
6283 }
6284
6285 /* There's a funny hw issue where the hw returns all 0 when reading from
6286  * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
6287  * ourselves, instead of doing a rmw cycle (which might result in us clearing
6288  * all limits and the gpu stuck at whatever frequency it is at atm).
6289  */
6290 static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
6291 {
6292         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6293         u32 limits;
6294
6295         /* Only set the down limit when we've reached the lowest level to avoid
6296          * getting more interrupts, otherwise leave this clear. This prevents a
6297          * race in the hw when coming out of rc6: There's a tiny window where
6298          * the hw runs at the minimal clock before selecting the desired
6299          * frequency, if the down threshold expires in that window we will not
6300          * receive a down interrupt. */
6301         if (INTEL_GEN(dev_priv) >= 9) {
6302                 limits = (rps->max_freq_softlimit) << 23;
6303                 if (val <= rps->min_freq_softlimit)
6304                         limits |= (rps->min_freq_softlimit) << 14;
6305         } else {
6306                 limits = rps->max_freq_softlimit << 24;
6307                 if (val <= rps->min_freq_softlimit)
6308                         limits |= rps->min_freq_softlimit << 16;
6309         }
6310
6311         return limits;
6312 }
6313
6314 static void rps_set_power(struct drm_i915_private *dev_priv, int new_power)
6315 {
6316         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6317         u32 threshold_up = 0, threshold_down = 0; /* in % */
6318         u32 ei_up = 0, ei_down = 0;
6319
6320         lockdep_assert_held(&rps->power.mutex);
6321
6322         if (new_power == rps->power.mode)
6323                 return;
6324
6325         /* Note the units here are not exactly 1us, but 1280ns. */
6326         switch (new_power) {
6327         case LOW_POWER:
6328                 /* Upclock if more than 95% busy over 16ms */
6329                 ei_up = 16000;
6330                 threshold_up = 95;
6331
6332                 /* Downclock if less than 85% busy over 32ms */
6333                 ei_down = 32000;
6334                 threshold_down = 85;
6335                 break;
6336
6337         case BETWEEN:
6338                 /* Upclock if more than 90% busy over 13ms */
6339                 ei_up = 13000;
6340                 threshold_up = 90;
6341
6342                 /* Downclock if less than 75% busy over 32ms */
6343                 ei_down = 32000;
6344                 threshold_down = 75;
6345                 break;
6346
6347         case HIGH_POWER:
6348                 /* Upclock if more than 85% busy over 10ms */
6349                 ei_up = 10000;
6350                 threshold_up = 85;
6351
6352                 /* Downclock if less than 60% busy over 32ms */
6353                 ei_down = 32000;
6354                 threshold_down = 60;
6355                 break;
6356         }
6357
6358         /* When byt can survive without system hang with dynamic
6359          * sw freq adjustments, this restriction can be lifted.
6360          */
6361         if (IS_VALLEYVIEW(dev_priv))
6362                 goto skip_hw_write;
6363
6364         I915_WRITE(GEN6_RP_UP_EI,
6365                    GT_INTERVAL_FROM_US(dev_priv, ei_up));
6366         I915_WRITE(GEN6_RP_UP_THRESHOLD,
6367                    GT_INTERVAL_FROM_US(dev_priv,
6368                                        ei_up * threshold_up / 100));
6369
6370         I915_WRITE(GEN6_RP_DOWN_EI,
6371                    GT_INTERVAL_FROM_US(dev_priv, ei_down));
6372         I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
6373                    GT_INTERVAL_FROM_US(dev_priv,
6374                                        ei_down * threshold_down / 100));
6375
6376         I915_WRITE(GEN6_RP_CONTROL,
6377                    GEN6_RP_MEDIA_TURBO |
6378                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
6379                    GEN6_RP_MEDIA_IS_GFX |
6380                    GEN6_RP_ENABLE |
6381                    GEN6_RP_UP_BUSY_AVG |
6382                    GEN6_RP_DOWN_IDLE_AVG);
6383
6384 skip_hw_write:
6385         rps->power.mode = new_power;
6386         rps->power.up_threshold = threshold_up;
6387         rps->power.down_threshold = threshold_down;
6388 }
6389
6390 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
6391 {
6392         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6393         int new_power;
6394
6395         new_power = rps->power.mode;
6396         switch (rps->power.mode) {
6397         case LOW_POWER:
6398                 if (val > rps->efficient_freq + 1 &&
6399                     val > rps->cur_freq)
6400                         new_power = BETWEEN;
6401                 break;
6402
6403         case BETWEEN:
6404                 if (val <= rps->efficient_freq &&
6405                     val < rps->cur_freq)
6406                         new_power = LOW_POWER;
6407                 else if (val >= rps->rp0_freq &&
6408                          val > rps->cur_freq)
6409                         new_power = HIGH_POWER;
6410                 break;
6411
6412         case HIGH_POWER:
6413                 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
6414                     val < rps->cur_freq)
6415                         new_power = BETWEEN;
6416                 break;
6417         }
6418         /* Max/min bins are special */
6419         if (val <= rps->min_freq_softlimit)
6420                 new_power = LOW_POWER;
6421         if (val >= rps->max_freq_softlimit)
6422                 new_power = HIGH_POWER;
6423
6424         mutex_lock(&rps->power.mutex);
6425         if (rps->power.interactive)
6426                 new_power = HIGH_POWER;
6427         rps_set_power(dev_priv, new_power);
6428         mutex_unlock(&rps->power.mutex);
6429         rps->last_adj = 0;
6430 }
6431
6432 void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive)
6433 {
6434         struct intel_rps *rps = &i915->gt_pm.rps;
6435
6436         if (INTEL_GEN(i915) < 6)
6437                 return;
6438
6439         mutex_lock(&rps->power.mutex);
6440         if (interactive) {
6441                 if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake))
6442                         rps_set_power(i915, HIGH_POWER);
6443         } else {
6444                 GEM_BUG_ON(!rps->power.interactive);
6445                 rps->power.interactive--;
6446         }
6447         mutex_unlock(&rps->power.mutex);
6448 }
6449
6450 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
6451 {
6452         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6453         u32 mask = 0;
6454
6455         /* We use UP_EI_EXPIRED interupts for both up/down in manual mode */
6456         if (val > rps->min_freq_softlimit)
6457                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
6458         if (val < rps->max_freq_softlimit)
6459                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
6460
6461         mask &= dev_priv->pm_rps_events;
6462
6463         return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
6464 }
6465
6466 /* gen6_set_rps is called to update the frequency request, but should also be
6467  * called when the range (min_delay and max_delay) is modified so that we can
6468  * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
6469 static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
6470 {
6471         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6472
6473         /* min/max delay may still have been modified so be sure to
6474          * write the limits value.
6475          */
6476         if (val != rps->cur_freq) {
6477                 gen6_set_rps_thresholds(dev_priv, val);
6478
6479                 if (INTEL_GEN(dev_priv) >= 9)
6480                         I915_WRITE(GEN6_RPNSWREQ,
6481                                    GEN9_FREQUENCY(val));
6482                 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6483                         I915_WRITE(GEN6_RPNSWREQ,
6484                                    HSW_FREQUENCY(val));
6485                 else
6486                         I915_WRITE(GEN6_RPNSWREQ,
6487                                    GEN6_FREQUENCY(val) |
6488                                    GEN6_OFFSET(0) |
6489                                    GEN6_AGGRESSIVE_TURBO);
6490         }
6491
6492         /* Make sure we continue to get interrupts
6493          * until we hit the minimum or maximum frequencies.
6494          */
6495         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
6496         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6497
6498         rps->cur_freq = val;
6499         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6500
6501         return 0;
6502 }
6503
6504 static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
6505 {
6506         int err;
6507
6508         if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
6509                       "Odd GPU freq value\n"))
6510                 val &= ~1;
6511
6512         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6513
6514         if (val != dev_priv->gt_pm.rps.cur_freq) {
6515                 err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
6516                 if (err)
6517                         return err;
6518
6519                 gen6_set_rps_thresholds(dev_priv, val);
6520         }
6521
6522         dev_priv->gt_pm.rps.cur_freq = val;
6523         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6524
6525         return 0;
6526 }
6527
6528 /* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
6529  *
6530  * * If Gfx is Idle, then
6531  * 1. Forcewake Media well.
6532  * 2. Request idle freq.
6533  * 3. Release Forcewake of Media well.
6534 */
6535 static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
6536 {
6537         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6538         u32 val = rps->idle_freq;
6539         int err;
6540
6541         if (rps->cur_freq <= val)
6542                 return;
6543
6544         /* The punit delays the write of the frequency and voltage until it
6545          * determines the GPU is awake. During normal usage we don't want to
6546          * waste power changing the frequency if the GPU is sleeping (rc6).
6547          * However, the GPU and driver is now idle and we do not want to delay
6548          * switching to minimum voltage (reducing power whilst idle) as we do
6549          * not expect to be woken in the near future and so must flush the
6550          * change by waking the device.
6551          *
6552          * We choose to take the media powerwell (either would do to trick the
6553          * punit into committing the voltage change) as that takes a lot less
6554          * power than the render powerwell.
6555          */
6556         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
6557         err = valleyview_set_rps(dev_priv, val);
6558         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
6559
6560         if (err)
6561                 DRM_ERROR("Failed to set RPS for idle\n");
6562 }
6563
6564 void gen6_rps_busy(struct drm_i915_private *dev_priv)
6565 {
6566         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6567
6568         mutex_lock(&dev_priv->pcu_lock);
6569         if (rps->enabled) {
6570                 u8 freq;
6571
6572                 if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
6573                         gen6_rps_reset_ei(dev_priv);
6574                 I915_WRITE(GEN6_PMINTRMSK,
6575                            gen6_rps_pm_mask(dev_priv, rps->cur_freq));
6576
6577                 gen6_enable_rps_interrupts(dev_priv);
6578
6579                 /* Use the user's desired frequency as a guide, but for better
6580                  * performance, jump directly to RPe as our starting frequency.
6581                  */
6582                 freq = max(rps->cur_freq,
6583                            rps->efficient_freq);
6584
6585                 if (intel_set_rps(dev_priv,
6586                                   clamp(freq,
6587                                         rps->min_freq_softlimit,
6588                                         rps->max_freq_softlimit)))
6589                         DRM_DEBUG_DRIVER("Failed to set idle frequency\n");
6590         }
6591         mutex_unlock(&dev_priv->pcu_lock);
6592 }
6593
6594 void gen6_rps_idle(struct drm_i915_private *dev_priv)
6595 {
6596         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6597
6598         /* Flush our bottom-half so that it does not race with us
6599          * setting the idle frequency and so that it is bounded by
6600          * our rpm wakeref. And then disable the interrupts to stop any
6601          * futher RPS reclocking whilst we are asleep.
6602          */
6603         gen6_disable_rps_interrupts(dev_priv);
6604
6605         mutex_lock(&dev_priv->pcu_lock);
6606         if (rps->enabled) {
6607                 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6608                         vlv_set_rps_idle(dev_priv);
6609                 else
6610                         gen6_set_rps(dev_priv, rps->idle_freq);
6611                 rps->last_adj = 0;
6612                 I915_WRITE(GEN6_PMINTRMSK,
6613                            gen6_sanitize_rps_pm_mask(dev_priv, ~0));
6614         }
6615         mutex_unlock(&dev_priv->pcu_lock);
6616 }
6617
6618 void gen6_rps_boost(struct i915_request *rq,
6619                     struct intel_rps_client *rps_client)
6620 {
6621         struct intel_rps *rps = &rq->i915->gt_pm.rps;
6622         unsigned long flags;
6623         bool boost;
6624
6625         /* This is intentionally racy! We peek at the state here, then
6626          * validate inside the RPS worker.
6627          */
6628         if (!rps->enabled)
6629                 return;
6630
6631         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
6632                 return;
6633
6634         /* Serializes with i915_request_retire() */
6635         boost = false;
6636         spin_lock_irqsave(&rq->lock, flags);
6637         if (!rq->waitboost && !dma_fence_is_signaled_locked(&rq->fence)) {
6638                 boost = !atomic_fetch_inc(&rps->num_waiters);
6639                 rq->waitboost = true;
6640         }
6641         spin_unlock_irqrestore(&rq->lock, flags);
6642         if (!boost)
6643                 return;
6644
6645         if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
6646                 schedule_work(&rps->work);
6647
6648         atomic_inc(rps_client ? &rps_client->boosts : &rps->boosts);
6649 }
6650
6651 int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
6652 {
6653         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6654         int err;
6655
6656         lockdep_assert_held(&dev_priv->pcu_lock);
6657         GEM_BUG_ON(val > rps->max_freq);
6658         GEM_BUG_ON(val < rps->min_freq);
6659
6660         if (!rps->enabled) {
6661                 rps->cur_freq = val;
6662                 return 0;
6663         }
6664
6665         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6666                 err = valleyview_set_rps(dev_priv, val);
6667         else
6668                 err = gen6_set_rps(dev_priv, val);
6669
6670         return err;
6671 }
6672
6673 static void gen9_disable_rc6(struct drm_i915_private *dev_priv)
6674 {
6675         I915_WRITE(GEN6_RC_CONTROL, 0);
6676         I915_WRITE(GEN9_PG_ENABLE, 0);
6677 }
6678
6679 static void gen9_disable_rps(struct drm_i915_private *dev_priv)
6680 {
6681         I915_WRITE(GEN6_RP_CONTROL, 0);
6682 }
6683
6684 static void gen6_disable_rc6(struct drm_i915_private *dev_priv)
6685 {
6686         I915_WRITE(GEN6_RC_CONTROL, 0);
6687 }
6688
6689 static void gen6_disable_rps(struct drm_i915_private *dev_priv)
6690 {
6691         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
6692         I915_WRITE(GEN6_RP_CONTROL, 0);
6693 }
6694
6695 static void cherryview_disable_rc6(struct drm_i915_private *dev_priv)
6696 {
6697         I915_WRITE(GEN6_RC_CONTROL, 0);
6698 }
6699
6700 static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
6701 {
6702         I915_WRITE(GEN6_RP_CONTROL, 0);
6703 }
6704
6705 static void valleyview_disable_rc6(struct drm_i915_private *dev_priv)
6706 {
6707         /* We're doing forcewake before Disabling RC6,
6708          * This what the BIOS expects when going into suspend */
6709         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6710
6711         I915_WRITE(GEN6_RC_CONTROL, 0);
6712
6713         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6714 }
6715
6716 static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
6717 {
6718         I915_WRITE(GEN6_RP_CONTROL, 0);
6719 }
6720
6721 static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
6722 {
6723         bool enable_rc6 = true;
6724         unsigned long rc6_ctx_base;
6725         u32 rc_ctl;
6726         int rc_sw_target;
6727
6728         rc_ctl = I915_READ(GEN6_RC_CONTROL);
6729         rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >>
6730                        RC_SW_TARGET_STATE_SHIFT;
6731         DRM_DEBUG_DRIVER("BIOS enabled RC states: "
6732                          "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
6733                          onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
6734                          onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
6735                          rc_sw_target);
6736
6737         if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
6738                 DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
6739                 enable_rc6 = false;
6740         }
6741
6742         /*
6743          * The exact context size is not known for BXT, so assume a page size
6744          * for this check.
6745          */
6746         rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
6747         if (!((rc6_ctx_base >= dev_priv->dsm_reserved.start) &&
6748               (rc6_ctx_base + PAGE_SIZE < dev_priv->dsm_reserved.end))) {
6749                 DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
6750                 enable_rc6 = false;
6751         }
6752
6753         if (!(((I915_READ(PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1) &&
6754               ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
6755               ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
6756               ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
6757                 DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
6758                 enable_rc6 = false;
6759         }
6760
6761         if (!I915_READ(GEN8_PUSHBUS_CONTROL) ||
6762             !I915_READ(GEN8_PUSHBUS_ENABLE) ||
6763             !I915_READ(GEN8_PUSHBUS_SHIFT)) {
6764                 DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
6765                 enable_rc6 = false;
6766         }
6767
6768         if (!I915_READ(GEN6_GFXPAUSE)) {
6769                 DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
6770                 enable_rc6 = false;
6771         }
6772
6773         if (!I915_READ(GEN8_MISC_CTRL0)) {
6774                 DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
6775                 enable_rc6 = false;
6776         }
6777
6778         return enable_rc6;
6779 }
6780
6781 static bool sanitize_rc6(struct drm_i915_private *i915)
6782 {
6783         struct intel_device_info *info = mkwrite_device_info(i915);
6784
6785         /* Powersaving is controlled by the host when inside a VM */
6786         if (intel_vgpu_active(i915))
6787                 info->has_rc6 = 0;
6788
6789         if (info->has_rc6 &&
6790             IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(i915)) {
6791                 DRM_INFO("RC6 disabled by BIOS\n");
6792                 info->has_rc6 = 0;
6793         }
6794
6795         /*
6796          * We assume that we do not have any deep rc6 levels if we don't have
6797          * have the previous rc6 level supported, i.e. we use HAS_RC6()
6798          * as the initial coarse check for rc6 in general, moving on to
6799          * progressively finer/deeper levels.
6800          */
6801         if (!info->has_rc6 && info->has_rc6p)
6802                 info->has_rc6p = 0;
6803
6804         return info->has_rc6;
6805 }
6806
6807 static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
6808 {
6809         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6810
6811         /* All of these values are in units of 50MHz */
6812
6813         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
6814         if (IS_GEN9_LP(dev_priv)) {
6815                 u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
6816                 rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
6817                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
6818                 rps->min_freq = (rp_state_cap >>  0) & 0xff;
6819         } else {
6820                 u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
6821                 rps->rp0_freq = (rp_state_cap >>  0) & 0xff;
6822                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
6823                 rps->min_freq = (rp_state_cap >> 16) & 0xff;
6824         }
6825         /* hw_max = RP0 until we check for overclocking */
6826         rps->max_freq = rps->rp0_freq;
6827
6828         rps->efficient_freq = rps->rp1_freq;
6829         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
6830             IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
6831                 u32 ddcc_status = 0;
6832
6833                 if (sandybridge_pcode_read(dev_priv,
6834                                            HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
6835                                            &ddcc_status) == 0)
6836                         rps->efficient_freq =
6837                                 clamp_t(u8,
6838                                         ((ddcc_status >> 8) & 0xff),
6839                                         rps->min_freq,
6840                                         rps->max_freq);
6841         }
6842
6843         if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
6844                 /* Store the frequency values in 16.66 MHZ units, which is
6845                  * the natural hardware unit for SKL
6846                  */
6847                 rps->rp0_freq *= GEN9_FREQ_SCALER;
6848                 rps->rp1_freq *= GEN9_FREQ_SCALER;
6849                 rps->min_freq *= GEN9_FREQ_SCALER;
6850                 rps->max_freq *= GEN9_FREQ_SCALER;
6851                 rps->efficient_freq *= GEN9_FREQ_SCALER;
6852         }
6853 }
6854
6855 static void reset_rps(struct drm_i915_private *dev_priv,
6856                       int (*set)(struct drm_i915_private *, u8))
6857 {
6858         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6859         u8 freq = rps->cur_freq;
6860
6861         /* force a reset */
6862         rps->power.mode = -1;
6863         rps->cur_freq = -1;
6864
6865         if (set(dev_priv, freq))
6866                 DRM_ERROR("Failed to reset RPS to initial values\n");
6867 }
6868
6869 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
6870 static void gen9_enable_rps(struct drm_i915_private *dev_priv)
6871 {
6872         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6873
6874         /* Program defaults and thresholds for RPS */
6875         if (IS_GEN9(dev_priv))
6876                 I915_WRITE(GEN6_RC_VIDEO_FREQ,
6877                         GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));
6878
6879         /* 1 second timeout*/
6880         I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
6881                 GT_INTERVAL_FROM_US(dev_priv, 1000000));
6882
6883         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
6884
6885         /* Leaning on the below call to gen6_set_rps to program/setup the
6886          * Up/Down EI & threshold registers, as well as the RP_CONTROL,
6887          * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
6888         reset_rps(dev_priv, gen6_set_rps);
6889
6890         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6891 }
6892
6893 static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
6894 {
6895         struct intel_engine_cs *engine;
6896         enum intel_engine_id id;
6897         u32 rc6_mode;
6898
6899         /* 1a: Software RC state - RC0 */
6900         I915_WRITE(GEN6_RC_STATE, 0);
6901
6902         /* 1b: Get forcewake during program sequence. Although the driver
6903          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6904         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6905
6906         /* 2a: Disable RC states. */
6907         I915_WRITE(GEN6_RC_CONTROL, 0);
6908
6909         /* 2b: Program RC6 thresholds.*/
6910         if (INTEL_GEN(dev_priv) >= 10) {
6911                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
6912                 I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
6913         } else if (IS_SKYLAKE(dev_priv)) {
6914                 /*
6915                  * WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only
6916                  * when CPG is enabled
6917                  */
6918                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
6919         } else {
6920                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
6921         }
6922
6923         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
6924         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
6925         for_each_engine(engine, dev_priv, id)
6926                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6927
6928         if (HAS_GUC(dev_priv))
6929                 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
6930
6931         I915_WRITE(GEN6_RC_SLEEP, 0);
6932
6933         /*
6934          * 2c: Program Coarse Power Gating Policies.
6935          *
6936          * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we
6937          * use instead is a more conservative estimate for the maximum time
6938          * it takes us to service a CS interrupt and submit a new ELSP - that
6939          * is the time which the GPU is idle waiting for the CPU to select the
6940          * next request to execute. If the idle hysteresis is less than that
6941          * interrupt service latency, the hardware will automatically gate
6942          * the power well and we will then incur the wake up cost on top of
6943          * the service latency. A similar guide from intel_pstate is that we
6944          * do not want the enable hysteresis to less than the wakeup latency.
6945          *
6946          * igt/gem_exec_nop/sequential provides a rough estimate for the
6947          * service latency, and puts it around 10us for Broadwell (and other
6948          * big core) and around 40us for Broxton (and other low power cores).
6949          * [Note that for legacy ringbuffer submission, this is less than 1us!]
6950          * However, the wakeup latency on Broxton is closer to 100us. To be
6951          * conservative, we have to factor in a context switch on top (due
6952          * to ksoftirqd).
6953          */
6954         I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
6955         I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
6956
6957         /* 3a: Enable RC6 */
6958         I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
6959
6960         /* WaRsUseTimeoutMode:cnl (pre-prod) */
6961         if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_C0))
6962                 rc6_mode = GEN7_RC_CTL_TO_MODE;
6963         else
6964                 rc6_mode = GEN6_RC_CTL_EI_MODE(1);
6965
6966         I915_WRITE(GEN6_RC_CONTROL,
6967                    GEN6_RC_CTL_HW_ENABLE |
6968                    GEN6_RC_CTL_RC6_ENABLE |
6969                    rc6_mode);
6970
6971         /*
6972          * 3b: Enable Coarse Power Gating only when RC6 is enabled.
6973          * WaRsDisableCoarsePowerGating:skl,cnl - Render/Media PG need to be disabled with RC6.
6974          */
6975         if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
6976                 I915_WRITE(GEN9_PG_ENABLE, 0);
6977         else
6978                 I915_WRITE(GEN9_PG_ENABLE,
6979                            GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
6980
6981         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6982 }
6983
6984 static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
6985 {
6986         struct intel_engine_cs *engine;
6987         enum intel_engine_id id;
6988
6989         /* 1a: Software RC state - RC0 */
6990         I915_WRITE(GEN6_RC_STATE, 0);
6991
6992         /* 1b: Get forcewake during program sequence. Although the driver
6993          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6994         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6995
6996         /* 2a: Disable RC states. */
6997         I915_WRITE(GEN6_RC_CONTROL, 0);
6998
6999         /* 2b: Program RC6 thresholds.*/
7000         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
7001         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
7002         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
7003         for_each_engine(engine, dev_priv, id)
7004                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7005         I915_WRITE(GEN6_RC_SLEEP, 0);
7006         I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
7007
7008         /* 3: Enable RC6 */
7009
7010         I915_WRITE(GEN6_RC_CONTROL,
7011                    GEN6_RC_CTL_HW_ENABLE |
7012                    GEN7_RC_CTL_TO_MODE |
7013                    GEN6_RC_CTL_RC6_ENABLE);
7014
7015         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7016 }
7017
7018 static void gen8_enable_rps(struct drm_i915_private *dev_priv)
7019 {
7020         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7021
7022         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7023
7024         /* 1 Program defaults and thresholds for RPS*/
7025         I915_WRITE(GEN6_RPNSWREQ,
7026                    HSW_FREQUENCY(rps->rp1_freq));
7027         I915_WRITE(GEN6_RC_VIDEO_FREQ,
7028                    HSW_FREQUENCY(rps->rp1_freq));
7029         /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
7030         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
7031
7032         /* Docs recommend 900MHz, and 300 MHz respectively */
7033         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
7034                    rps->max_freq_softlimit << 24 |
7035                    rps->min_freq_softlimit << 16);
7036
7037         I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
7038         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
7039         I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
7040         I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
7041
7042         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7043
7044         /* 2: Enable RPS */
7045         I915_WRITE(GEN6_RP_CONTROL,
7046                    GEN6_RP_MEDIA_TURBO |
7047                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
7048                    GEN6_RP_MEDIA_IS_GFX |
7049                    GEN6_RP_ENABLE |
7050                    GEN6_RP_UP_BUSY_AVG |
7051                    GEN6_RP_DOWN_IDLE_AVG);
7052
7053         reset_rps(dev_priv, gen6_set_rps);
7054
7055         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7056 }
7057
7058 static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
7059 {
7060         struct intel_engine_cs *engine;
7061         enum intel_engine_id id;
7062         u32 rc6vids, rc6_mask;
7063         u32 gtfifodbg;
7064         int ret;
7065
7066         I915_WRITE(GEN6_RC_STATE, 0);
7067
7068         /* Clear the DBG now so we don't confuse earlier errors */
7069         gtfifodbg = I915_READ(GTFIFODBG);
7070         if (gtfifodbg) {
7071                 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
7072                 I915_WRITE(GTFIFODBG, gtfifodbg);
7073         }
7074
7075         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7076
7077         /* disable the counters and set deterministic thresholds */
7078         I915_WRITE(GEN6_RC_CONTROL, 0);
7079
7080         I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
7081         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
7082         I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
7083         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7084         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7085
7086         for_each_engine(engine, dev_priv, id)
7087                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7088
7089         I915_WRITE(GEN6_RC_SLEEP, 0);
7090         I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
7091         if (IS_IVYBRIDGE(dev_priv))
7092                 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
7093         else
7094                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
7095         I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
7096         I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
7097
7098         /* We don't use those on Haswell */
7099         rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
7100         if (HAS_RC6p(dev_priv))
7101                 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
7102         if (HAS_RC6pp(dev_priv))
7103                 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
7104         I915_WRITE(GEN6_RC_CONTROL,
7105                    rc6_mask |
7106                    GEN6_RC_CTL_EI_MODE(1) |
7107                    GEN6_RC_CTL_HW_ENABLE);
7108
7109         rc6vids = 0;
7110         ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
7111         if (IS_GEN6(dev_priv) && ret) {
7112                 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
7113         } else if (IS_GEN6(dev_priv) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
7114                 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
7115                           GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
7116                 rc6vids &= 0xffff00;
7117                 rc6vids |= GEN6_ENCODE_RC6_VID(450);
7118                 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
7119                 if (ret)
7120                         DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
7121         }
7122
7123         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7124 }
7125
7126 static void gen6_enable_rps(struct drm_i915_private *dev_priv)
7127 {
7128         /* Here begins a magic sequence of register writes to enable
7129          * auto-downclocking.
7130          *
7131          * Perhaps there might be some value in exposing these to
7132          * userspace...
7133          */
7134         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7135
7136         /* Power down if completely idle for over 50ms */
7137         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
7138         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7139
7140         reset_rps(dev_priv, gen6_set_rps);
7141
7142         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7143 }
7144
7145 static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
7146 {
7147         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7148         const int min_freq = 15;
7149         const int scaling_factor = 180;
7150         unsigned int gpu_freq;
7151         unsigned int max_ia_freq, min_ring_freq;
7152         unsigned int max_gpu_freq, min_gpu_freq;
7153         struct cpufreq_policy *policy;
7154
7155         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
7156
7157         if (rps->max_freq <= rps->min_freq)
7158                 return;
7159
7160         policy = cpufreq_cpu_get(0);
7161         if (policy) {
7162                 max_ia_freq = policy->cpuinfo.max_freq;
7163                 cpufreq_cpu_put(policy);
7164         } else {
7165                 /*
7166                  * Default to measured freq if none found, PCU will ensure we
7167                  * don't go over
7168                  */
7169                 max_ia_freq = tsc_khz;
7170         }
7171
7172         /* Convert from kHz to MHz */
7173         max_ia_freq /= 1000;
7174
7175         min_ring_freq = I915_READ(DCLK) & 0xf;
7176         /* convert DDR frequency from units of 266.6MHz to bandwidth */
7177         min_ring_freq = mult_frac(min_ring_freq, 8, 3);
7178
7179         min_gpu_freq = rps->min_freq;
7180         max_gpu_freq = rps->max_freq;
7181         if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7182                 /* Convert GT frequency to 50 HZ units */
7183                 min_gpu_freq /= GEN9_FREQ_SCALER;
7184                 max_gpu_freq /= GEN9_FREQ_SCALER;
7185         }
7186
7187         /*
7188          * For each potential GPU frequency, load a ring frequency we'd like
7189          * to use for memory access.  We do this by specifying the IA frequency
7190          * the PCU should use as a reference to determine the ring frequency.
7191          */
7192         for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
7193                 const int diff = max_gpu_freq - gpu_freq;
7194                 unsigned int ia_freq = 0, ring_freq = 0;
7195
7196                 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7197                         /*
7198                          * ring_freq = 2 * GT. ring_freq is in 100MHz units
7199                          * No floor required for ring frequency on SKL.
7200                          */
7201                         ring_freq = gpu_freq;
7202                 } else if (INTEL_GEN(dev_priv) >= 8) {
7203                         /* max(2 * GT, DDR). NB: GT is 50MHz units */
7204                         ring_freq = max(min_ring_freq, gpu_freq);
7205                 } else if (IS_HASWELL(dev_priv)) {
7206                         ring_freq = mult_frac(gpu_freq, 5, 4);
7207                         ring_freq = max(min_ring_freq, ring_freq);
7208                         /* leave ia_freq as the default, chosen by cpufreq */
7209                 } else {
7210                         /* On older processors, there is no separate ring
7211                          * clock domain, so in order to boost the bandwidth
7212                          * of the ring, we need to upclock the CPU (ia_freq).
7213                          *
7214                          * For GPU frequencies less than 750MHz,
7215                          * just use the lowest ring freq.
7216                          */
7217                         if (gpu_freq < min_freq)
7218                                 ia_freq = 800;
7219                         else
7220                                 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
7221                         ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
7222                 }
7223
7224                 sandybridge_pcode_write(dev_priv,
7225                                         GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
7226                                         ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
7227                                         ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
7228                                         gpu_freq);
7229         }
7230 }
7231
7232 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
7233 {
7234         u32 val, rp0;
7235
7236         val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7237
7238         switch (INTEL_INFO(dev_priv)->sseu.eu_total) {
7239         case 8:
7240                 /* (2 * 4) config */
7241                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
7242                 break;
7243         case 12:
7244                 /* (2 * 6) config */
7245                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
7246                 break;
7247         case 16:
7248                 /* (2 * 8) config */
7249         default:
7250                 /* Setting (2 * 8) Min RP0 for any other combination */
7251                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
7252                 break;
7253         }
7254
7255         rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
7256
7257         return rp0;
7258 }
7259
7260 static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7261 {
7262         u32 val, rpe;
7263
7264         val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
7265         rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
7266
7267         return rpe;
7268 }
7269
7270 static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
7271 {
7272         u32 val, rp1;
7273
7274         val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7275         rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
7276
7277         return rp1;
7278 }
7279
7280 static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
7281 {
7282         u32 val, rpn;
7283
7284         val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
7285         rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
7286                        FB_GFX_FREQ_FUSE_MASK);
7287
7288         return rpn;
7289 }
7290
7291 static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
7292 {
7293         u32 val, rp1;
7294
7295         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7296
7297         rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
7298
7299         return rp1;
7300 }
7301
7302 static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
7303 {
7304         u32 val, rp0;
7305
7306         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7307
7308         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
7309         /* Clamp to max */
7310         rp0 = min_t(u32, rp0, 0xea);
7311
7312         return rp0;
7313 }
7314
7315 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7316 {
7317         u32 val, rpe;
7318
7319         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
7320         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
7321         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
7322         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
7323
7324         return rpe;
7325 }
7326
7327 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
7328 {
7329         u32 val;
7330
7331         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
7332         /*
7333          * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
7334          * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
7335          * a BYT-M B0 the above register contains 0xbf. Moreover when setting
7336          * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
7337          * to make sure it matches what Punit accepts.
7338          */
7339         return max_t(u32, val, 0xc0);
7340 }
7341
7342 /* Check that the pctx buffer wasn't move under us. */
7343 static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
7344 {
7345         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7346
7347         WARN_ON(pctx_addr != dev_priv->dsm.start +
7348                              dev_priv->vlv_pctx->stolen->start);
7349 }
7350
7351
7352 /* Check that the pcbr address is not empty. */
7353 static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
7354 {
7355         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7356
7357         WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
7358 }
7359
7360 static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
7361 {
7362         resource_size_t pctx_paddr, paddr;
7363         resource_size_t pctx_size = 32*1024;
7364         u32 pcbr;
7365
7366         pcbr = I915_READ(VLV_PCBR);
7367         if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
7368                 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7369                 paddr = dev_priv->dsm.end + 1 - pctx_size;
7370                 GEM_BUG_ON(paddr > U32_MAX);
7371
7372                 pctx_paddr = (paddr & (~4095));
7373                 I915_WRITE(VLV_PCBR, pctx_paddr);
7374         }
7375
7376         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7377 }
7378
7379 static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
7380 {
7381         struct drm_i915_gem_object *pctx;
7382         resource_size_t pctx_paddr;
7383         resource_size_t pctx_size = 24*1024;
7384         u32 pcbr;
7385
7386         pcbr = I915_READ(VLV_PCBR);
7387         if (pcbr) {
7388                 /* BIOS set it up already, grab the pre-alloc'd space */
7389                 resource_size_t pcbr_offset;
7390
7391                 pcbr_offset = (pcbr & (~4095)) - dev_priv->dsm.start;
7392                 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv,
7393                                                                       pcbr_offset,
7394                                                                       I915_GTT_OFFSET_NONE,
7395                                                                       pctx_size);
7396                 goto out;
7397         }
7398
7399         DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7400
7401         /*
7402          * From the Gunit register HAS:
7403          * The Gfx driver is expected to program this register and ensure
7404          * proper allocation within Gfx stolen memory.  For example, this
7405          * register should be programmed such than the PCBR range does not
7406          * overlap with other ranges, such as the frame buffer, protected
7407          * memory, or any other relevant ranges.
7408          */
7409         pctx = i915_gem_object_create_stolen(dev_priv, pctx_size);
7410         if (!pctx) {
7411                 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
7412                 goto out;
7413         }
7414
7415         GEM_BUG_ON(range_overflows_t(u64,
7416                                      dev_priv->dsm.start,
7417                                      pctx->stolen->start,
7418                                      U32_MAX));
7419         pctx_paddr = dev_priv->dsm.start + pctx->stolen->start;
7420         I915_WRITE(VLV_PCBR, pctx_paddr);
7421
7422 out:
7423         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7424         dev_priv->vlv_pctx = pctx;
7425 }
7426
7427 static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
7428 {
7429         struct drm_i915_gem_object *pctx;
7430
7431         pctx = fetch_and_zero(&dev_priv->vlv_pctx);
7432         if (pctx)
7433                 i915_gem_object_put(pctx);
7434 }
7435
7436 static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
7437 {
7438         dev_priv->gt_pm.rps.gpll_ref_freq =
7439                 vlv_get_cck_clock(dev_priv, "GPLL ref",
7440                                   CCK_GPLL_CLOCK_CONTROL,
7441                                   dev_priv->czclk_freq);
7442
7443         DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n",
7444                          dev_priv->gt_pm.rps.gpll_ref_freq);
7445 }
7446
7447 static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
7448 {
7449         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7450         u32 val;
7451
7452         valleyview_setup_pctx(dev_priv);
7453
7454         vlv_init_gpll_ref_freq(dev_priv);
7455
7456         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7457         switch ((val >> 6) & 3) {
7458         case 0:
7459         case 1:
7460                 dev_priv->mem_freq = 800;
7461                 break;
7462         case 2:
7463                 dev_priv->mem_freq = 1066;
7464                 break;
7465         case 3:
7466                 dev_priv->mem_freq = 1333;
7467                 break;
7468         }
7469         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7470
7471         rps->max_freq = valleyview_rps_max_freq(dev_priv);
7472         rps->rp0_freq = rps->max_freq;
7473         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7474                          intel_gpu_freq(dev_priv, rps->max_freq),
7475                          rps->max_freq);
7476
7477         rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv);
7478         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7479                          intel_gpu_freq(dev_priv, rps->efficient_freq),
7480                          rps->efficient_freq);
7481
7482         rps->rp1_freq = valleyview_rps_guar_freq(dev_priv);
7483         DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
7484                          intel_gpu_freq(dev_priv, rps->rp1_freq),
7485                          rps->rp1_freq);
7486
7487         rps->min_freq = valleyview_rps_min_freq(dev_priv);
7488         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7489                          intel_gpu_freq(dev_priv, rps->min_freq),
7490                          rps->min_freq);
7491 }
7492
7493 static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
7494 {
7495         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7496         u32 val;
7497
7498         cherryview_setup_pctx(dev_priv);
7499
7500         vlv_init_gpll_ref_freq(dev_priv);
7501
7502         mutex_lock(&dev_priv->sb_lock);
7503         val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
7504         mutex_unlock(&dev_priv->sb_lock);
7505
7506         switch ((val >> 2) & 0x7) {
7507         case 3:
7508                 dev_priv->mem_freq = 2000;
7509                 break;
7510         default:
7511                 dev_priv->mem_freq = 1600;
7512                 break;
7513         }
7514         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7515
7516         rps->max_freq = cherryview_rps_max_freq(dev_priv);
7517         rps->rp0_freq = rps->max_freq;
7518         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7519                          intel_gpu_freq(dev_priv, rps->max_freq),
7520                          rps->max_freq);
7521
7522         rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv);
7523         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7524                          intel_gpu_freq(dev_priv, rps->efficient_freq),
7525                          rps->efficient_freq);
7526
7527         rps->rp1_freq = cherryview_rps_guar_freq(dev_priv);
7528         DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
7529                          intel_gpu_freq(dev_priv, rps->rp1_freq),
7530                          rps->rp1_freq);
7531
7532         rps->min_freq = cherryview_rps_min_freq(dev_priv);
7533         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7534                          intel_gpu_freq(dev_priv, rps->min_freq),
7535                          rps->min_freq);
7536
7537         WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
7538                    rps->min_freq) & 1,
7539                   "Odd GPU freq values\n");
7540 }
7541
7542 static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
7543 {
7544         valleyview_cleanup_pctx(dev_priv);
7545 }
7546
7547 static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
7548 {
7549         struct intel_engine_cs *engine;
7550         enum intel_engine_id id;
7551         u32 gtfifodbg, rc6_mode, pcbr;
7552
7553         gtfifodbg = I915_READ(GTFIFODBG) & ~(GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV |
7554                                              GT_FIFO_FREE_ENTRIES_CHV);
7555         if (gtfifodbg) {
7556                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7557                                  gtfifodbg);
7558                 I915_WRITE(GTFIFODBG, gtfifodbg);
7559         }
7560
7561         cherryview_check_pctx(dev_priv);
7562
7563         /* 1a & 1b: Get forcewake during program sequence. Although the driver
7564          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
7565         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7566
7567         /*  Disable RC states. */
7568         I915_WRITE(GEN6_RC_CONTROL, 0);
7569
7570         /* 2a: Program RC6 thresholds.*/
7571         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
7572         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
7573         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
7574
7575         for_each_engine(engine, dev_priv, id)
7576                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7577         I915_WRITE(GEN6_RC_SLEEP, 0);
7578
7579         /* TO threshold set to 500 us ( 0x186 * 1.28 us) */
7580         I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
7581
7582         /* Allows RC6 residency counter to work */
7583         I915_WRITE(VLV_COUNTER_CONTROL,
7584                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7585                                       VLV_MEDIA_RC6_COUNT_EN |
7586                                       VLV_RENDER_RC6_COUNT_EN));
7587
7588         /* For now we assume BIOS is allocating and populating the PCBR  */
7589         pcbr = I915_READ(VLV_PCBR);
7590
7591         /* 3: Enable RC6 */
7592         rc6_mode = 0;
7593         if (pcbr >> VLV_PCBR_ADDR_SHIFT)
7594                 rc6_mode = GEN7_RC_CTL_TO_MODE;
7595         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
7596
7597         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7598 }
7599
7600 static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
7601 {
7602         u32 val;
7603
7604         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7605
7606         /* 1: Program defaults and thresholds for RPS*/
7607         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7608         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7609         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7610         I915_WRITE(GEN6_RP_UP_EI, 66000);
7611         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7612
7613         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7614
7615         /* 2: Enable RPS */
7616         I915_WRITE(GEN6_RP_CONTROL,
7617                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
7618                    GEN6_RP_MEDIA_IS_GFX |
7619                    GEN6_RP_ENABLE |
7620                    GEN6_RP_UP_BUSY_AVG |
7621                    GEN6_RP_DOWN_IDLE_AVG);
7622
7623         /* Setting Fixed Bias */
7624         val = VLV_OVERRIDE_EN |
7625                   VLV_SOC_TDP_EN |
7626                   CHV_BIAS_CPU_50_SOC_50;
7627         vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7628
7629         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7630
7631         /* RPS code assumes GPLL is used */
7632         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7633
7634         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7635         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7636
7637         reset_rps(dev_priv, valleyview_set_rps);
7638
7639         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7640 }
7641
7642 static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
7643 {
7644         struct intel_engine_cs *engine;
7645         enum intel_engine_id id;
7646         u32 gtfifodbg;
7647
7648         valleyview_check_pctx(dev_priv);
7649
7650         gtfifodbg = I915_READ(GTFIFODBG);
7651         if (gtfifodbg) {
7652                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7653                                  gtfifodbg);
7654                 I915_WRITE(GTFIFODBG, gtfifodbg);
7655         }
7656
7657         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7658
7659         /*  Disable RC states. */
7660         I915_WRITE(GEN6_RC_CONTROL, 0);
7661
7662         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
7663         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7664         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7665
7666         for_each_engine(engine, dev_priv, id)
7667                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7668
7669         I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
7670
7671         /* Allows RC6 residency counter to work */
7672         I915_WRITE(VLV_COUNTER_CONTROL,
7673                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7674                                       VLV_MEDIA_RC0_COUNT_EN |
7675                                       VLV_RENDER_RC0_COUNT_EN |
7676                                       VLV_MEDIA_RC6_COUNT_EN |
7677                                       VLV_RENDER_RC6_COUNT_EN));
7678
7679         I915_WRITE(GEN6_RC_CONTROL,
7680                    GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL);
7681
7682         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7683 }
7684
7685 static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
7686 {
7687         u32 val;
7688
7689         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7690
7691         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7692         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7693         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7694         I915_WRITE(GEN6_RP_UP_EI, 66000);
7695         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7696
7697         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7698
7699         I915_WRITE(GEN6_RP_CONTROL,
7700                    GEN6_RP_MEDIA_TURBO |
7701                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
7702                    GEN6_RP_MEDIA_IS_GFX |
7703                    GEN6_RP_ENABLE |
7704                    GEN6_RP_UP_BUSY_AVG |
7705                    GEN6_RP_DOWN_IDLE_CONT);
7706
7707         /* Setting Fixed Bias */
7708         val = VLV_OVERRIDE_EN |
7709                   VLV_SOC_TDP_EN |
7710                   VLV_BIAS_CPU_125_SOC_875;
7711         vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7712
7713         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7714
7715         /* RPS code assumes GPLL is used */
7716         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7717
7718         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7719         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7720
7721         reset_rps(dev_priv, valleyview_set_rps);
7722
7723         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7724 }
7725
7726 static unsigned long intel_pxfreq(u32 vidfreq)
7727 {
7728         unsigned long freq;
7729         int div = (vidfreq & 0x3f0000) >> 16;
7730         int post = (vidfreq & 0x3000) >> 12;
7731         int pre = (vidfreq & 0x7);
7732
7733         if (!pre)
7734                 return 0;
7735
7736         freq = ((div * 133333) / ((1<<post) * pre));
7737
7738         return freq;
7739 }
7740
7741 static const struct cparams {
7742         u16 i;
7743         u16 t;
7744         u16 m;
7745         u16 c;
7746 } cparams[] = {
7747         { 1, 1333, 301, 28664 },
7748         { 1, 1066, 294, 24460 },
7749         { 1, 800, 294, 25192 },
7750         { 0, 1333, 276, 27605 },
7751         { 0, 1066, 276, 27605 },
7752         { 0, 800, 231, 23784 },
7753 };
7754
7755 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
7756 {
7757         u64 total_count, diff, ret;
7758         u32 count1, count2, count3, m = 0, c = 0;
7759         unsigned long now = jiffies_to_msecs(jiffies), diff1;
7760         int i;
7761
7762         lockdep_assert_held(&mchdev_lock);
7763
7764         diff1 = now - dev_priv->ips.last_time1;
7765
7766         /* Prevent division-by-zero if we are asking too fast.
7767          * Also, we don't get interesting results if we are polling
7768          * faster than once in 10ms, so just return the saved value
7769          * in such cases.
7770          */
7771         if (diff1 <= 10)
7772                 return dev_priv->ips.chipset_power;
7773
7774         count1 = I915_READ(DMIEC);
7775         count2 = I915_READ(DDREC);
7776         count3 = I915_READ(CSIEC);
7777
7778         total_count = count1 + count2 + count3;
7779
7780         /* FIXME: handle per-counter overflow */
7781         if (total_count < dev_priv->ips.last_count1) {
7782                 diff = ~0UL - dev_priv->ips.last_count1;
7783                 diff += total_count;
7784         } else {
7785                 diff = total_count - dev_priv->ips.last_count1;
7786         }
7787
7788         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
7789                 if (cparams[i].i == dev_priv->ips.c_m &&
7790                     cparams[i].t == dev_priv->ips.r_t) {
7791                         m = cparams[i].m;
7792                         c = cparams[i].c;
7793                         break;
7794                 }
7795         }
7796
7797         diff = div_u64(diff, diff1);
7798         ret = ((m * diff) + c);
7799         ret = div_u64(ret, 10);
7800
7801         dev_priv->ips.last_count1 = total_count;
7802         dev_priv->ips.last_time1 = now;
7803
7804         dev_priv->ips.chipset_power = ret;
7805
7806         return ret;
7807 }
7808
7809 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
7810 {
7811         unsigned long val;
7812
7813         if (!IS_GEN5(dev_priv))
7814                 return 0;
7815
7816         spin_lock_irq(&mchdev_lock);
7817
7818         val = __i915_chipset_val(dev_priv);
7819
7820         spin_unlock_irq(&mchdev_lock);
7821
7822         return val;
7823 }
7824
7825 unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
7826 {
7827         unsigned long m, x, b;
7828         u32 tsfs;
7829
7830         tsfs = I915_READ(TSFS);
7831
7832         m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
7833         x = I915_READ8(TR1);
7834
7835         b = tsfs & TSFS_INTR_MASK;
7836
7837         return ((m * x) / 127) - b;
7838 }
7839
7840 static int _pxvid_to_vd(u8 pxvid)
7841 {
7842         if (pxvid == 0)
7843                 return 0;
7844
7845         if (pxvid >= 8 && pxvid < 31)
7846                 pxvid = 31;
7847
7848         return (pxvid + 2) * 125;
7849 }
7850
7851 static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
7852 {
7853         const int vd = _pxvid_to_vd(pxvid);
7854         const int vm = vd - 1125;
7855
7856         if (INTEL_INFO(dev_priv)->is_mobile)
7857                 return vm > 0 ? vm : 0;
7858
7859         return vd;
7860 }
7861
7862 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
7863 {
7864         u64 now, diff, diffms;
7865         u32 count;
7866
7867         lockdep_assert_held(&mchdev_lock);
7868
7869         now = ktime_get_raw_ns();
7870         diffms = now - dev_priv->ips.last_time2;
7871         do_div(diffms, NSEC_PER_MSEC);
7872
7873         /* Don't divide by 0 */
7874         if (!diffms)
7875                 return;
7876
7877         count = I915_READ(GFXEC);
7878
7879         if (count < dev_priv->ips.last_count2) {
7880                 diff = ~0UL - dev_priv->ips.last_count2;
7881                 diff += count;
7882         } else {
7883                 diff = count - dev_priv->ips.last_count2;
7884         }
7885
7886         dev_priv->ips.last_count2 = count;
7887         dev_priv->ips.last_time2 = now;
7888
7889         /* More magic constants... */
7890         diff = diff * 1181;
7891         diff = div_u64(diff, diffms * 10);
7892         dev_priv->ips.gfx_power = diff;
7893 }
7894
7895 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
7896 {
7897         if (!IS_GEN5(dev_priv))
7898                 return;
7899
7900         spin_lock_irq(&mchdev_lock);
7901
7902         __i915_update_gfx_val(dev_priv);
7903
7904         spin_unlock_irq(&mchdev_lock);
7905 }
7906
7907 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
7908 {
7909         unsigned long t, corr, state1, corr2, state2;
7910         u32 pxvid, ext_v;
7911
7912         lockdep_assert_held(&mchdev_lock);
7913
7914         pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq));
7915         pxvid = (pxvid >> 24) & 0x7f;
7916         ext_v = pvid_to_extvid(dev_priv, pxvid);
7917
7918         state1 = ext_v;
7919
7920         t = i915_mch_val(dev_priv);
7921
7922         /* Revel in the empirically derived constants */
7923
7924         /* Correction factor in 1/100000 units */
7925         if (t > 80)
7926                 corr = ((t * 2349) + 135940);
7927         else if (t >= 50)
7928                 corr = ((t * 964) + 29317);
7929         else /* < 50 */
7930                 corr = ((t * 301) + 1004);
7931
7932         corr = corr * ((150142 * state1) / 10000 - 78642);
7933         corr /= 100000;
7934         corr2 = (corr * dev_priv->ips.corr);
7935
7936         state2 = (corr2 * state1) / 10000;
7937         state2 /= 100; /* convert to mW */
7938
7939         __i915_update_gfx_val(dev_priv);
7940
7941         return dev_priv->ips.gfx_power + state2;
7942 }
7943
7944 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
7945 {
7946         unsigned long val;
7947
7948         if (!IS_GEN5(dev_priv))
7949                 return 0;
7950
7951         spin_lock_irq(&mchdev_lock);
7952
7953         val = __i915_gfx_val(dev_priv);
7954
7955         spin_unlock_irq(&mchdev_lock);
7956
7957         return val;
7958 }
7959
7960 /**
7961  * i915_read_mch_val - return value for IPS use
7962  *
7963  * Calculate and return a value for the IPS driver to use when deciding whether
7964  * we have thermal and power headroom to increase CPU or GPU power budget.
7965  */
7966 unsigned long i915_read_mch_val(void)
7967 {
7968         struct drm_i915_private *dev_priv;
7969         unsigned long chipset_val, graphics_val, ret = 0;
7970
7971         spin_lock_irq(&mchdev_lock);
7972         if (!i915_mch_dev)
7973                 goto out_unlock;
7974         dev_priv = i915_mch_dev;
7975
7976         chipset_val = __i915_chipset_val(dev_priv);
7977         graphics_val = __i915_gfx_val(dev_priv);
7978
7979         ret = chipset_val + graphics_val;
7980
7981 out_unlock:
7982         spin_unlock_irq(&mchdev_lock);
7983
7984         return ret;
7985 }
7986 EXPORT_SYMBOL_GPL(i915_read_mch_val);
7987
7988 /**
7989  * i915_gpu_raise - raise GPU frequency limit
7990  *
7991  * Raise the limit; IPS indicates we have thermal headroom.
7992  */
7993 bool i915_gpu_raise(void)
7994 {
7995         struct drm_i915_private *dev_priv;
7996         bool ret = true;
7997
7998         spin_lock_irq(&mchdev_lock);
7999         if (!i915_mch_dev) {
8000                 ret = false;
8001                 goto out_unlock;
8002         }
8003         dev_priv = i915_mch_dev;
8004
8005         if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
8006                 dev_priv->ips.max_delay--;
8007
8008 out_unlock:
8009         spin_unlock_irq(&mchdev_lock);
8010
8011         return ret;
8012 }
8013 EXPORT_SYMBOL_GPL(i915_gpu_raise);
8014
8015 /**
8016  * i915_gpu_lower - lower GPU frequency limit
8017  *
8018  * IPS indicates we're close to a thermal limit, so throttle back the GPU
8019  * frequency maximum.
8020  */
8021 bool i915_gpu_lower(void)
8022 {
8023         struct drm_i915_private *dev_priv;
8024         bool ret = true;
8025
8026         spin_lock_irq(&mchdev_lock);
8027         if (!i915_mch_dev) {
8028                 ret = false;
8029                 goto out_unlock;
8030         }
8031         dev_priv = i915_mch_dev;
8032
8033         if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
8034                 dev_priv->ips.max_delay++;
8035
8036 out_unlock:
8037         spin_unlock_irq(&mchdev_lock);
8038
8039         return ret;
8040 }
8041 EXPORT_SYMBOL_GPL(i915_gpu_lower);
8042
8043 /**
8044  * i915_gpu_busy - indicate GPU business to IPS
8045  *
8046  * Tell the IPS driver whether or not the GPU is busy.
8047  */
8048 bool i915_gpu_busy(void)
8049 {
8050         bool ret = false;
8051
8052         spin_lock_irq(&mchdev_lock);
8053         if (i915_mch_dev)
8054                 ret = i915_mch_dev->gt.awake;
8055         spin_unlock_irq(&mchdev_lock);
8056
8057         return ret;
8058 }
8059 EXPORT_SYMBOL_GPL(i915_gpu_busy);
8060
8061 /**
8062  * i915_gpu_turbo_disable - disable graphics turbo
8063  *
8064  * Disable graphics turbo by resetting the max frequency and setting the
8065  * current frequency to the default.
8066  */
8067 bool i915_gpu_turbo_disable(void)
8068 {
8069         struct drm_i915_private *dev_priv;
8070         bool ret = true;
8071
8072         spin_lock_irq(&mchdev_lock);
8073         if (!i915_mch_dev) {
8074                 ret = false;
8075                 goto out_unlock;
8076         }
8077         dev_priv = i915_mch_dev;
8078
8079         dev_priv->ips.max_delay = dev_priv->ips.fstart;
8080
8081         if (!ironlake_set_drps(dev_priv, dev_priv->ips.fstart))
8082                 ret = false;
8083
8084 out_unlock:
8085         spin_unlock_irq(&mchdev_lock);
8086
8087         return ret;
8088 }
8089 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
8090
8091 /**
8092  * Tells the intel_ips driver that the i915 driver is now loaded, if
8093  * IPS got loaded first.
8094  *
8095  * This awkward dance is so that neither module has to depend on the
8096  * other in order for IPS to do the appropriate communication of
8097  * GPU turbo limits to i915.
8098  */
8099 static void
8100 ips_ping_for_i915_load(void)
8101 {
8102         void (*link)(void);
8103
8104         link = symbol_get(ips_link_to_i915_driver);
8105         if (link) {
8106                 link();
8107                 symbol_put(ips_link_to_i915_driver);
8108         }
8109 }
8110
8111 void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
8112 {
8113         /* We only register the i915 ips part with intel-ips once everything is
8114          * set up, to avoid intel-ips sneaking in and reading bogus values. */
8115         spin_lock_irq(&mchdev_lock);
8116         i915_mch_dev = dev_priv;
8117         spin_unlock_irq(&mchdev_lock);
8118
8119         ips_ping_for_i915_load();
8120 }
8121
8122 void intel_gpu_ips_teardown(void)
8123 {
8124         spin_lock_irq(&mchdev_lock);
8125         i915_mch_dev = NULL;
8126         spin_unlock_irq(&mchdev_lock);
8127 }
8128
8129 static void intel_init_emon(struct drm_i915_private *dev_priv)
8130 {
8131         u32 lcfuse;
8132         u8 pxw[16];
8133         int i;
8134
8135         /* Disable to program */
8136         I915_WRITE(ECR, 0);
8137         POSTING_READ(ECR);
8138
8139         /* Program energy weights for various events */
8140         I915_WRITE(SDEW, 0x15040d00);
8141         I915_WRITE(CSIEW0, 0x007f0000);
8142         I915_WRITE(CSIEW1, 0x1e220004);
8143         I915_WRITE(CSIEW2, 0x04000004);
8144
8145         for (i = 0; i < 5; i++)
8146                 I915_WRITE(PEW(i), 0);
8147         for (i = 0; i < 3; i++)
8148                 I915_WRITE(DEW(i), 0);
8149
8150         /* Program P-state weights to account for frequency power adjustment */
8151         for (i = 0; i < 16; i++) {
8152                 u32 pxvidfreq = I915_READ(PXVFREQ(i));
8153                 unsigned long freq = intel_pxfreq(pxvidfreq);
8154                 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
8155                         PXVFREQ_PX_SHIFT;
8156                 unsigned long val;
8157
8158                 val = vid * vid;
8159                 val *= (freq / 1000);
8160                 val *= 255;
8161                 val /= (127*127*900);
8162                 if (val > 0xff)
8163                         DRM_ERROR("bad pxval: %ld\n", val);
8164                 pxw[i] = val;
8165         }
8166         /* Render standby states get 0 weight */
8167         pxw[14] = 0;
8168         pxw[15] = 0;
8169
8170         for (i = 0; i < 4; i++) {
8171                 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
8172                         (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
8173                 I915_WRITE(PXW(i), val);
8174         }
8175
8176         /* Adjust magic regs to magic values (more experimental results) */
8177         I915_WRITE(OGW0, 0);
8178         I915_WRITE(OGW1, 0);
8179         I915_WRITE(EG0, 0x00007f00);
8180         I915_WRITE(EG1, 0x0000000e);
8181         I915_WRITE(EG2, 0x000e0000);
8182         I915_WRITE(EG3, 0x68000300);
8183         I915_WRITE(EG4, 0x42000000);
8184         I915_WRITE(EG5, 0x00140031);
8185         I915_WRITE(EG6, 0);
8186         I915_WRITE(EG7, 0);
8187
8188         for (i = 0; i < 8; i++)
8189                 I915_WRITE(PXWL(i), 0);
8190
8191         /* Enable PMON + select events */
8192         I915_WRITE(ECR, 0x80000019);
8193
8194         lcfuse = I915_READ(LCFUSE02);
8195
8196         dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
8197 }
8198
8199 static bool i915_rc6_ctx_corrupted(struct drm_i915_private *dev_priv)
8200 {
8201         return !I915_READ(GEN8_RC6_CTX_INFO);
8202 }
8203
8204 static void i915_rc6_ctx_wa_init(struct drm_i915_private *i915)
8205 {
8206         if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915))
8207                 return;
8208
8209         if (i915_rc6_ctx_corrupted(i915)) {
8210                 DRM_INFO("RC6 context corrupted, disabling runtime power management\n");
8211                 i915->gt_pm.rc6.ctx_corrupted = true;
8212                 intel_runtime_pm_get(i915);
8213         }
8214 }
8215
8216 static void i915_rc6_ctx_wa_cleanup(struct drm_i915_private *i915)
8217 {
8218         if (i915->gt_pm.rc6.ctx_corrupted) {
8219                 intel_runtime_pm_put(i915);
8220                 i915->gt_pm.rc6.ctx_corrupted = false;
8221         }
8222 }
8223
8224 /**
8225  * i915_rc6_ctx_wa_suspend - system suspend sequence for the RC6 CTX WA
8226  * @i915: i915 device
8227  *
8228  * Perform any steps needed to clean up the RC6 CTX WA before system suspend.
8229  */
8230 void i915_rc6_ctx_wa_suspend(struct drm_i915_private *i915)
8231 {
8232         if (i915->gt_pm.rc6.ctx_corrupted)
8233                 intel_runtime_pm_put(i915);
8234 }
8235
8236 /**
8237  * i915_rc6_ctx_wa_resume - system resume sequence for the RC6 CTX WA
8238  * @i915: i915 device
8239  *
8240  * Perform any steps needed to re-init the RC6 CTX WA after system resume.
8241  */
8242 void i915_rc6_ctx_wa_resume(struct drm_i915_private *i915)
8243 {
8244         if (!i915->gt_pm.rc6.ctx_corrupted)
8245                 return;
8246
8247         if (i915_rc6_ctx_corrupted(i915)) {
8248                 intel_runtime_pm_get(i915);
8249                 return;
8250         }
8251
8252         DRM_INFO("RC6 context restored, re-enabling runtime power management\n");
8253         i915->gt_pm.rc6.ctx_corrupted = false;
8254 }
8255
8256 static void intel_disable_rc6(struct drm_i915_private *dev_priv);
8257
8258 /**
8259  * i915_rc6_ctx_wa_check - check for a new RC6 CTX corruption
8260  * @i915: i915 device
8261  *
8262  * Check if an RC6 CTX corruption has happened since the last check and if so
8263  * disable RC6 and runtime power management.
8264  *
8265  * Return false if no context corruption has happened since the last call of
8266  * this function, true otherwise.
8267 */
8268 bool i915_rc6_ctx_wa_check(struct drm_i915_private *i915)
8269 {
8270         if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915))
8271                 return false;
8272
8273         if (i915->gt_pm.rc6.ctx_corrupted)
8274                 return false;
8275
8276         if (!i915_rc6_ctx_corrupted(i915))
8277                 return false;
8278
8279         DRM_NOTE("RC6 context corruption, disabling runtime power management\n");
8280
8281         intel_disable_rc6(i915);
8282         i915->gt_pm.rc6.ctx_corrupted = true;
8283         intel_runtime_pm_get_noresume(i915);
8284
8285         return true;
8286 }
8287
8288 void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
8289 {
8290         struct intel_rps *rps = &dev_priv->gt_pm.rps;
8291
8292         /*
8293          * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
8294          * requirement.
8295          */
8296         if (!sanitize_rc6(dev_priv)) {
8297                 DRM_INFO("RC6 disabled, disabling runtime PM support\n");
8298                 intel_runtime_pm_get(dev_priv);
8299         }
8300
8301         mutex_lock(&dev_priv->pcu_lock);
8302
8303         i915_rc6_ctx_wa_init(dev_priv);
8304
8305         /* Initialize RPS limits (for userspace) */
8306         if (IS_CHERRYVIEW(dev_priv))
8307                 cherryview_init_gt_powersave(dev_priv);
8308         else if (IS_VALLEYVIEW(dev_priv))
8309                 valleyview_init_gt_powersave(dev_priv);
8310         else if (INTEL_GEN(dev_priv) >= 6)
8311                 gen6_init_rps_frequencies(dev_priv);
8312
8313         /* Derive initial user preferences/limits from the hardware limits */
8314         rps->idle_freq = rps->min_freq;
8315         rps->cur_freq = rps->idle_freq;
8316
8317         rps->max_freq_softlimit = rps->max_freq;
8318         rps->min_freq_softlimit = rps->min_freq;
8319
8320         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
8321                 rps->min_freq_softlimit =
8322                         max_t(int,
8323                               rps->efficient_freq,
8324                               intel_freq_opcode(dev_priv, 450));
8325
8326         /* After setting max-softlimit, find the overclock max freq */
8327         if (IS_GEN6(dev_priv) ||
8328             IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
8329                 u32 params = 0;
8330
8331                 sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &params);
8332                 if (params & BIT(31)) { /* OC supported */
8333                         DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
8334                                          (rps->max_freq & 0xff) * 50,
8335                                          (params & 0xff) * 50);
8336                         rps->max_freq = params & 0xff;
8337                 }
8338         }
8339
8340         /* Finally allow us to boost to max by default */
8341         rps->boost_freq = rps->max_freq;
8342
8343         mutex_unlock(&dev_priv->pcu_lock);
8344 }
8345
8346 void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
8347 {
8348         if (IS_VALLEYVIEW(dev_priv))
8349                 valleyview_cleanup_gt_powersave(dev_priv);
8350
8351         i915_rc6_ctx_wa_cleanup(dev_priv);
8352
8353         if (!HAS_RC6(dev_priv))
8354                 intel_runtime_pm_put(dev_priv);
8355 }
8356
8357 /**
8358  * intel_suspend_gt_powersave - suspend PM work and helper threads
8359  * @dev_priv: i915 device
8360  *
8361  * We don't want to disable RC6 or other features here, we just want
8362  * to make sure any work we've queued has finished and won't bother
8363  * us while we're suspended.
8364  */
8365 void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
8366 {
8367         if (INTEL_GEN(dev_priv) < 6)
8368                 return;
8369
8370         /* gen6_rps_idle() will be called later to disable interrupts */
8371 }
8372
8373 void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
8374 {
8375         dev_priv->gt_pm.rps.enabled = true; /* force RPS disabling */
8376         dev_priv->gt_pm.rc6.enabled = true; /* force RC6 disabling */
8377         intel_disable_gt_powersave(dev_priv);
8378
8379         if (INTEL_GEN(dev_priv) >= 11)
8380                 gen11_reset_rps_interrupts(dev_priv);
8381         else
8382                 gen6_reset_rps_interrupts(dev_priv);
8383 }
8384
8385 static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
8386 {
8387         lockdep_assert_held(&i915->pcu_lock);
8388
8389         if (!i915->gt_pm.llc_pstate.enabled)
8390                 return;
8391
8392         /* Currently there is no HW configuration to be done to disable. */
8393
8394         i915->gt_pm.llc_pstate.enabled = false;
8395 }
8396
8397 static void __intel_disable_rc6(struct drm_i915_private *dev_priv)
8398 {
8399         lockdep_assert_held(&dev_priv->pcu_lock);
8400
8401         if (!dev_priv->gt_pm.rc6.enabled)
8402                 return;
8403
8404         if (INTEL_GEN(dev_priv) >= 9)
8405                 gen9_disable_rc6(dev_priv);
8406         else if (IS_CHERRYVIEW(dev_priv))
8407                 cherryview_disable_rc6(dev_priv);
8408         else if (IS_VALLEYVIEW(dev_priv))
8409                 valleyview_disable_rc6(dev_priv);
8410         else if (INTEL_GEN(dev_priv) >= 6)
8411                 gen6_disable_rc6(dev_priv);
8412
8413         dev_priv->gt_pm.rc6.enabled = false;
8414 }
8415
8416 static void intel_disable_rc6(struct drm_i915_private *dev_priv)
8417 {
8418         mutex_lock(&dev_priv->pcu_lock);
8419         __intel_disable_rc6(dev_priv);
8420         mutex_unlock(&dev_priv->pcu_lock);
8421 }
8422
8423 static void intel_disable_rps(struct drm_i915_private *dev_priv)
8424 {
8425         lockdep_assert_held(&dev_priv->pcu_lock);
8426
8427         if (!dev_priv->gt_pm.rps.enabled)
8428                 return;
8429
8430         if (INTEL_GEN(dev_priv) >= 9)
8431                 gen9_disable_rps(dev_priv);
8432         else if (IS_CHERRYVIEW(dev_priv))
8433                 cherryview_disable_rps(dev_priv);
8434         else if (IS_VALLEYVIEW(dev_priv))
8435                 valleyview_disable_rps(dev_priv);
8436         else if (INTEL_GEN(dev_priv) >= 6)
8437                 gen6_disable_rps(dev_priv);
8438         else if (IS_IRONLAKE_M(dev_priv))
8439                 ironlake_disable_drps(dev_priv);
8440
8441         dev_priv->gt_pm.rps.enabled = false;
8442 }
8443
8444 void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
8445 {
8446         mutex_lock(&dev_priv->pcu_lock);
8447
8448         __intel_disable_rc6(dev_priv);
8449         intel_disable_rps(dev_priv);
8450         if (HAS_LLC(dev_priv))
8451                 intel_disable_llc_pstate(dev_priv);
8452
8453         mutex_unlock(&dev_priv->pcu_lock);
8454 }
8455
8456 static inline void intel_enable_llc_pstate(struct drm_i915_private *i915)
8457 {
8458         lockdep_assert_held(&i915->pcu_lock);
8459
8460         if (i915->gt_pm.llc_pstate.enabled)
8461                 return;
8462
8463         gen6_update_ring_freq(i915);
8464
8465         i915->gt_pm.llc_pstate.enabled = true;
8466 }
8467
8468 static void intel_enable_rc6(struct drm_i915_private *dev_priv)
8469 {
8470         lockdep_assert_held(&dev_priv->pcu_lock);
8471
8472         if (dev_priv->gt_pm.rc6.enabled)
8473                 return;
8474
8475         if (dev_priv->gt_pm.rc6.ctx_corrupted)
8476                 return;
8477
8478         if (IS_CHERRYVIEW(dev_priv))
8479                 cherryview_enable_rc6(dev_priv);
8480         else if (IS_VALLEYVIEW(dev_priv))
8481                 valleyview_enable_rc6(dev_priv);
8482         else if (INTEL_GEN(dev_priv) >= 9)
8483                 gen9_enable_rc6(dev_priv);
8484         else if (IS_BROADWELL(dev_priv))
8485                 gen8_enable_rc6(dev_priv);
8486         else if (INTEL_GEN(dev_priv) >= 6)
8487                 gen6_enable_rc6(dev_priv);
8488
8489         dev_priv->gt_pm.rc6.enabled = true;
8490 }
8491
8492 static void intel_enable_rps(struct drm_i915_private *dev_priv)
8493 {
8494         struct intel_rps *rps = &dev_priv->gt_pm.rps;
8495
8496         lockdep_assert_held(&dev_priv->pcu_lock);
8497
8498         if (rps->enabled)
8499                 return;
8500
8501         if (IS_CHERRYVIEW(dev_priv)) {
8502                 cherryview_enable_rps(dev_priv);
8503         } else if (IS_VALLEYVIEW(dev_priv)) {
8504                 valleyview_enable_rps(dev_priv);
8505         } else if (INTEL_GEN(dev_priv) >= 9) {
8506                 gen9_enable_rps(dev_priv);
8507         } else if (IS_BROADWELL(dev_priv)) {
8508                 gen8_enable_rps(dev_priv);
8509         } else if (INTEL_GEN(dev_priv) >= 6) {
8510                 gen6_enable_rps(dev_priv);
8511         } else if (IS_IRONLAKE_M(dev_priv)) {
8512                 ironlake_enable_drps(dev_priv);
8513                 intel_init_emon(dev_priv);
8514         }
8515
8516         WARN_ON(rps->max_freq < rps->min_freq);
8517         WARN_ON(rps->idle_freq > rps->max_freq);
8518
8519         WARN_ON(rps->efficient_freq < rps->min_freq);
8520         WARN_ON(rps->efficient_freq > rps->max_freq);
8521
8522         rps->enabled = true;
8523 }
8524
8525 void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
8526 {
8527         /* Powersaving is controlled by the host when inside a VM */
8528         if (intel_vgpu_active(dev_priv))
8529                 return;
8530
8531         mutex_lock(&dev_priv->pcu_lock);
8532
8533         if (HAS_RC6(dev_priv))
8534                 intel_enable_rc6(dev_priv);
8535         intel_enable_rps(dev_priv);
8536         if (HAS_LLC(dev_priv))
8537                 intel_enable_llc_pstate(dev_priv);
8538
8539         mutex_unlock(&dev_priv->pcu_lock);
8540 }
8541
8542 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
8543 {
8544         /*
8545          * On Ibex Peak and Cougar Point, we need to disable clock
8546          * gating for the panel power sequencer or it will fail to
8547          * start up when no ports are active.
8548          */
8549         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
8550 }
8551
8552 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
8553 {
8554         enum pipe pipe;
8555
8556         for_each_pipe(dev_priv, pipe) {
8557                 I915_WRITE(DSPCNTR(pipe),
8558                            I915_READ(DSPCNTR(pipe)) |
8559                            DISPPLANE_TRICKLE_FEED_DISABLE);
8560
8561                 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
8562                 POSTING_READ(DSPSURF(pipe));
8563         }
8564 }
8565
8566 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
8567 {
8568         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8569
8570         /*
8571          * Required for FBC
8572          * WaFbcDisableDpfcClockGating:ilk
8573          */
8574         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
8575                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
8576                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
8577
8578         I915_WRITE(PCH_3DCGDIS0,
8579                    MARIUNIT_CLOCK_GATE_DISABLE |
8580                    SVSMUNIT_CLOCK_GATE_DISABLE);
8581         I915_WRITE(PCH_3DCGDIS1,
8582                    VFMUNIT_CLOCK_GATE_DISABLE);
8583
8584         /*
8585          * According to the spec the following bits should be set in
8586          * order to enable memory self-refresh
8587          * The bit 22/21 of 0x42004
8588          * The bit 5 of 0x42020
8589          * The bit 15 of 0x45000
8590          */
8591         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8592                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
8593                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
8594         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
8595         I915_WRITE(DISP_ARB_CTL,
8596                    (I915_READ(DISP_ARB_CTL) |
8597                     DISP_FBC_WM_DIS));
8598
8599         /*
8600          * Based on the document from hardware guys the following bits
8601          * should be set unconditionally in order to enable FBC.
8602          * The bit 22 of 0x42000
8603          * The bit 22 of 0x42004
8604          * The bit 7,8,9 of 0x42020.
8605          */
8606         if (IS_IRONLAKE_M(dev_priv)) {
8607                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
8608                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
8609                            I915_READ(ILK_DISPLAY_CHICKEN1) |
8610                            ILK_FBCQ_DIS);
8611                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8612                            I915_READ(ILK_DISPLAY_CHICKEN2) |
8613                            ILK_DPARB_GATE);
8614         }
8615
8616         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8617
8618         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8619                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8620                    ILK_ELPIN_409_SELECT);
8621         I915_WRITE(_3D_CHICKEN2,
8622                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
8623                    _3D_CHICKEN2_WM_READ_PIPELINED);
8624
8625         /* WaDisableRenderCachePipelinedFlush:ilk */
8626         I915_WRITE(CACHE_MODE_0,
8627                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
8628
8629         /* WaDisable_RenderCache_OperationalFlush:ilk */
8630         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8631
8632         g4x_disable_trickle_feed(dev_priv);
8633
8634         ibx_init_clock_gating(dev_priv);
8635 }
8636
8637 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
8638 {
8639         int pipe;
8640         uint32_t val;
8641
8642         /*
8643          * On Ibex Peak and Cougar Point, we need to disable clock
8644          * gating for the panel power sequencer or it will fail to
8645          * start up when no ports are active.
8646          */
8647         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
8648                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
8649                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
8650         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
8651                    DPLS_EDP_PPS_FIX_DIS);
8652         /* The below fixes the weird display corruption, a few pixels shifted
8653          * downward, on (only) LVDS of some HP laptops with IVY.
8654          */
8655         for_each_pipe(dev_priv, pipe) {
8656                 val = I915_READ(TRANS_CHICKEN2(pipe));
8657                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
8658                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8659                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
8660                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8661                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
8662                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
8663                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
8664                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
8665         }
8666         /* WADP0ClockGatingDisable */
8667         for_each_pipe(dev_priv, pipe) {
8668                 I915_WRITE(TRANS_CHICKEN1(pipe),
8669                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
8670         }
8671 }
8672
8673 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
8674 {
8675         uint32_t tmp;
8676
8677         tmp = I915_READ(MCH_SSKPD);
8678         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
8679                 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
8680                               tmp);
8681 }
8682
8683 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
8684 {
8685         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8686
8687         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8688
8689         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8690                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8691                    ILK_ELPIN_409_SELECT);
8692
8693         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
8694         I915_WRITE(_3D_CHICKEN,
8695                    _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
8696
8697         /* WaDisable_RenderCache_OperationalFlush:snb */
8698         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8699
8700         /*
8701          * BSpec recoomends 8x4 when MSAA is used,
8702          * however in practice 16x4 seems fastest.
8703          *
8704          * Note that PS/WM thread counts depend on the WIZ hashing
8705          * disable bit, which we don't touch here, but it's good
8706          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
8707          */
8708         I915_WRITE(GEN6_GT_MODE,
8709                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8710
8711         I915_WRITE(CACHE_MODE_0,
8712                    _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
8713
8714         I915_WRITE(GEN6_UCGCTL1,
8715                    I915_READ(GEN6_UCGCTL1) |
8716                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
8717                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
8718
8719         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
8720          * gating disable must be set.  Failure to set it results in
8721          * flickering pixels due to Z write ordering failures after
8722          * some amount of runtime in the Mesa "fire" demo, and Unigine
8723          * Sanctuary and Tropics, and apparently anything else with
8724          * alpha test or pixel discard.
8725          *
8726          * According to the spec, bit 11 (RCCUNIT) must also be set,
8727          * but we didn't debug actual testcases to find it out.
8728          *
8729          * WaDisableRCCUnitClockGating:snb
8730          * WaDisableRCPBUnitClockGating:snb
8731          */
8732         I915_WRITE(GEN6_UCGCTL2,
8733                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
8734                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
8735
8736         /* WaStripsFansDisableFastClipPerformanceFix:snb */
8737         I915_WRITE(_3D_CHICKEN3,
8738                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
8739
8740         /*
8741          * Bspec says:
8742          * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
8743          * 3DSTATE_SF number of SF output attributes is more than 16."
8744          */
8745         I915_WRITE(_3D_CHICKEN3,
8746                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
8747
8748         /*
8749          * According to the spec the following bits should be
8750          * set in order to enable memory self-refresh and fbc:
8751          * The bit21 and bit22 of 0x42000
8752          * The bit21 and bit22 of 0x42004
8753          * The bit5 and bit7 of 0x42020
8754          * The bit14 of 0x70180
8755          * The bit14 of 0x71180
8756          *
8757          * WaFbcAsynchFlipDisableFbcQueue:snb
8758          */
8759         I915_WRITE(ILK_DISPLAY_CHICKEN1,
8760                    I915_READ(ILK_DISPLAY_CHICKEN1) |
8761                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
8762         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8763                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8764                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
8765         I915_WRITE(ILK_DSPCLK_GATE_D,
8766                    I915_READ(ILK_DSPCLK_GATE_D) |
8767                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
8768                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
8769
8770         g4x_disable_trickle_feed(dev_priv);
8771
8772         cpt_init_clock_gating(dev_priv);
8773
8774         gen6_check_mch_setup(dev_priv);
8775 }
8776
8777 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
8778 {
8779         uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
8780
8781         /*
8782          * WaVSThreadDispatchOverride:ivb,vlv
8783          *
8784          * This actually overrides the dispatch
8785          * mode for all thread types.
8786          */
8787         reg &= ~GEN7_FF_SCHED_MASK;
8788         reg |= GEN7_FF_TS_SCHED_HW;
8789         reg |= GEN7_FF_VS_SCHED_HW;
8790         reg |= GEN7_FF_DS_SCHED_HW;
8791
8792         I915_WRITE(GEN7_FF_THREAD_MODE, reg);
8793 }
8794
8795 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
8796 {
8797         /*
8798          * TODO: this bit should only be enabled when really needed, then
8799          * disabled when not needed anymore in order to save power.
8800          */
8801         if (HAS_PCH_LPT_LP(dev_priv))
8802                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
8803                            I915_READ(SOUTH_DSPCLK_GATE_D) |
8804                            PCH_LP_PARTITION_LEVEL_DISABLE);
8805
8806         /* WADPOClockGatingDisable:hsw */
8807         I915_WRITE(TRANS_CHICKEN1(PIPE_A),
8808                    I915_READ(TRANS_CHICKEN1(PIPE_A)) |
8809                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
8810 }
8811
8812 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
8813 {
8814         if (HAS_PCH_LPT_LP(dev_priv)) {
8815                 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
8816
8817                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
8818                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
8819         }
8820 }
8821
8822 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
8823                                    int general_prio_credits,
8824                                    int high_prio_credits)
8825 {
8826         u32 misccpctl;
8827         u32 val;
8828
8829         /* WaTempDisableDOPClkGating:bdw */
8830         misccpctl = I915_READ(GEN7_MISCCPCTL);
8831         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
8832
8833         val = I915_READ(GEN8_L3SQCREG1);
8834         val &= ~L3_PRIO_CREDITS_MASK;
8835         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
8836         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
8837         I915_WRITE(GEN8_L3SQCREG1, val);
8838
8839         /*
8840          * Wait at least 100 clocks before re-enabling clock gating.
8841          * See the definition of L3SQCREG1 in BSpec.
8842          */
8843         POSTING_READ(GEN8_L3SQCREG1);
8844         udelay(1);
8845         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
8846 }
8847
8848 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
8849 {
8850         /* This is not an Wa. Enable to reduce Sampler power */
8851         I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
8852                    I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
8853 }
8854
8855 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
8856 {
8857         if (!HAS_PCH_CNP(dev_priv))
8858                 return;
8859
8860         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
8861         I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
8862                    CNP_PWM_CGE_GATING_DISABLE);
8863 }
8864
8865 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
8866 {
8867         u32 val;
8868         cnp_init_clock_gating(dev_priv);
8869
8870         /* This is not an Wa. Enable for better image quality */
8871         I915_WRITE(_3D_CHICKEN3,
8872                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
8873
8874         /* WaEnableChickenDCPR:cnl */
8875         I915_WRITE(GEN8_CHICKEN_DCPR_1,
8876                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
8877
8878         /* WaFbcWakeMemOn:cnl */
8879         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
8880                    DISP_FBC_MEMORY_WAKE);
8881
8882         val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
8883         /* ReadHitWriteOnlyDisable:cnl */
8884         val |= RCCUNIT_CLKGATE_DIS;
8885         /* WaSarbUnitClockGatingDisable:cnl (pre-prod) */
8886         if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0))
8887                 val |= SARBUNIT_CLKGATE_DIS;
8888         I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
8889
8890         /* Wa_2201832410:cnl */
8891         val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
8892         val |= GWUNIT_CLKGATE_DIS;
8893         I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
8894
8895         /* WaDisableVFclkgate:cnl */
8896         /* WaVFUnitClockGatingDisable:cnl */
8897         val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
8898         val |= VFUNIT_CLKGATE_DIS;
8899         I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
8900 }
8901
8902 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
8903 {
8904         cnp_init_clock_gating(dev_priv);
8905         gen9_init_clock_gating(dev_priv);
8906
8907         /* WaFbcNukeOnHostModify:cfl */
8908         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8909                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8910 }
8911
8912 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
8913 {
8914         gen9_init_clock_gating(dev_priv);
8915
8916         /* WaDisableSDEUnitClockGating:kbl */
8917         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
8918                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
8919                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
8920
8921         /* WaDisableGamClockGating:kbl */
8922         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
8923                 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
8924                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
8925
8926         /* WaFbcNukeOnHostModify:kbl */
8927         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8928                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8929 }
8930
8931 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
8932 {
8933         gen9_init_clock_gating(dev_priv);
8934
8935         /* WAC6entrylatency:skl */
8936         I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
8937                    FBC_LLC_FULLY_OPEN);
8938
8939         /* WaFbcNukeOnHostModify:skl */
8940         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8941                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8942 }
8943
8944 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
8945 {
8946         /* The GTT cache must be disabled if the system is using 2M pages. */
8947         bool can_use_gtt_cache = !HAS_PAGE_SIZES(dev_priv,
8948                                                  I915_GTT_PAGE_SIZE_2M);
8949         enum pipe pipe;
8950
8951         /* WaSwitchSolVfFArbitrationPriority:bdw */
8952         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
8953
8954         /* WaPsrDPAMaskVBlankInSRD:bdw */
8955         I915_WRITE(CHICKEN_PAR1_1,
8956                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
8957
8958         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
8959         for_each_pipe(dev_priv, pipe) {
8960                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
8961                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
8962                            BDW_DPRS_MASK_VBLANK_SRD);
8963         }
8964
8965         /* WaVSRefCountFullforceMissDisable:bdw */
8966         /* WaDSRefCountFullforceMissDisable:bdw */
8967         I915_WRITE(GEN7_FF_THREAD_MODE,
8968                    I915_READ(GEN7_FF_THREAD_MODE) &
8969                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
8970
8971         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
8972                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
8973
8974         /* WaDisableSDEUnitClockGating:bdw */
8975         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
8976                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
8977
8978         /* WaProgramL3SqcReg1Default:bdw */
8979         gen8_set_l3sqc_credits(dev_priv, 30, 2);
8980
8981         /* WaGttCachingOffByDefault:bdw */
8982         I915_WRITE(HSW_GTT_CACHE_EN, can_use_gtt_cache ? GTT_CACHE_EN_ALL : 0);
8983
8984         /* WaKVMNotificationOnConfigChange:bdw */
8985         I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
8986                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
8987
8988         lpt_init_clock_gating(dev_priv);
8989
8990         /* WaDisableDopClockGating:bdw
8991          *
8992          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
8993          * clock gating.
8994          */
8995         I915_WRITE(GEN6_UCGCTL1,
8996                    I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
8997 }
8998
8999 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
9000 {
9001         /* L3 caching of data atomics doesn't work -- disable it. */
9002         I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
9003         I915_WRITE(HSW_ROW_CHICKEN3,
9004                    _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
9005
9006         /* This is required by WaCatErrorRejectionIssue:hsw */
9007         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9008                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9009                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9010
9011         /* WaVSRefCountFullforceMissDisable:hsw */
9012         I915_WRITE(GEN7_FF_THREAD_MODE,
9013                    I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
9014
9015         /* WaDisable_RenderCache_OperationalFlush:hsw */
9016         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9017
9018         /* enable HiZ Raw Stall Optimization */
9019         I915_WRITE(CACHE_MODE_0_GEN7,
9020                    _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
9021
9022         /* WaDisable4x2SubspanOptimization:hsw */
9023         I915_WRITE(CACHE_MODE_1,
9024                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9025
9026         /*
9027          * BSpec recommends 8x4 when MSAA is used,
9028          * however in practice 16x4 seems fastest.
9029          *
9030          * Note that PS/WM thread counts depend on the WIZ hashing
9031          * disable bit, which we don't touch here, but it's good
9032          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
9033          */
9034         I915_WRITE(GEN7_GT_MODE,
9035                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9036
9037         /* WaSampleCChickenBitEnable:hsw */
9038         I915_WRITE(HALF_SLICE_CHICKEN3,
9039                    _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
9040
9041         /* WaSwitchSolVfFArbitrationPriority:hsw */
9042         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
9043
9044         lpt_init_clock_gating(dev_priv);
9045 }
9046
9047 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
9048 {
9049         uint32_t snpcr;
9050
9051         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
9052
9053         /* WaDisableEarlyCull:ivb */
9054         I915_WRITE(_3D_CHICKEN3,
9055                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
9056
9057         /* WaDisableBackToBackFlipFix:ivb */
9058         I915_WRITE(IVB_CHICKEN3,
9059                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
9060                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
9061
9062         /* WaDisablePSDDualDispatchEnable:ivb */
9063         if (IS_IVB_GT1(dev_priv))
9064                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
9065                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
9066
9067         /* WaDisable_RenderCache_OperationalFlush:ivb */
9068         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9069
9070         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
9071         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
9072                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
9073
9074         /* WaApplyL3ControlAndL3ChickenMode:ivb */
9075         I915_WRITE(GEN7_L3CNTLREG1,
9076                         GEN7_WA_FOR_GEN7_L3_CONTROL);
9077         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
9078                    GEN7_WA_L3_CHICKEN_MODE);
9079         if (IS_IVB_GT1(dev_priv))
9080                 I915_WRITE(GEN7_ROW_CHICKEN2,
9081                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9082         else {
9083                 /* must write both registers */
9084                 I915_WRITE(GEN7_ROW_CHICKEN2,
9085                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9086                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
9087                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9088         }
9089
9090         /* WaForceL3Serialization:ivb */
9091         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
9092                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
9093
9094         /*
9095          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
9096          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
9097          */
9098         I915_WRITE(GEN6_UCGCTL2,
9099                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
9100
9101         /* This is required by WaCatErrorRejectionIssue:ivb */
9102         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9103                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9104                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9105
9106         g4x_disable_trickle_feed(dev_priv);
9107
9108         gen7_setup_fixed_func_scheduler(dev_priv);
9109
9110         if (0) { /* causes HiZ corruption on ivb:gt1 */
9111                 /* enable HiZ Raw Stall Optimization */
9112                 I915_WRITE(CACHE_MODE_0_GEN7,
9113                            _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
9114         }
9115
9116         /* WaDisable4x2SubspanOptimization:ivb */
9117         I915_WRITE(CACHE_MODE_1,
9118                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9119
9120         /*
9121          * BSpec recommends 8x4 when MSAA is used,
9122          * however in practice 16x4 seems fastest.
9123          *
9124          * Note that PS/WM thread counts depend on the WIZ hashing
9125          * disable bit, which we don't touch here, but it's good
9126          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
9127          */
9128         I915_WRITE(GEN7_GT_MODE,
9129                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9130
9131         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
9132         snpcr &= ~GEN6_MBC_SNPCR_MASK;
9133         snpcr |= GEN6_MBC_SNPCR_MED;
9134         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
9135
9136         if (!HAS_PCH_NOP(dev_priv))
9137                 cpt_init_clock_gating(dev_priv);
9138
9139         gen6_check_mch_setup(dev_priv);
9140 }
9141
9142 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
9143 {
9144         /* WaDisableEarlyCull:vlv */
9145         I915_WRITE(_3D_CHICKEN3,
9146                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
9147
9148         /* WaDisableBackToBackFlipFix:vlv */
9149         I915_WRITE(IVB_CHICKEN3,
9150                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
9151                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
9152
9153         /* WaPsdDispatchEnable:vlv */
9154         /* WaDisablePSDDualDispatchEnable:vlv */
9155         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
9156                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
9157                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
9158
9159         /* WaDisable_RenderCache_OperationalFlush:vlv */
9160         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9161
9162         /* WaForceL3Serialization:vlv */
9163         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
9164                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
9165
9166         /* WaDisableDopClockGating:vlv */
9167         I915_WRITE(GEN7_ROW_CHICKEN2,
9168                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9169
9170         /* This is required by WaCatErrorRejectionIssue:vlv */
9171         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9172                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9173                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9174
9175         gen7_setup_fixed_func_scheduler(dev_priv);
9176
9177         /*
9178          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
9179          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
9180          */
9181         I915_WRITE(GEN6_UCGCTL2,
9182                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
9183
9184         /* WaDisableL3Bank2xClockGate:vlv
9185          * Disabling L3 clock gating- MMIO 940c[25] = 1
9186          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
9187         I915_WRITE(GEN7_UCGCTL4,
9188                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
9189
9190         /*
9191          * BSpec says this must be set, even though
9192          * WaDisable4x2SubspanOptimization isn't listed for VLV.
9193          */
9194         I915_WRITE(CACHE_MODE_1,
9195                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9196
9197         /*
9198          * BSpec recommends 8x4 when MSAA is used,
9199          * however in practice 16x4 seems fastest.
9200          *
9201          * Note that PS/WM thread counts depend on the WIZ hashing
9202          * disable bit, which we don't touch here, but it's good
9203          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
9204          */
9205         I915_WRITE(GEN7_GT_MODE,
9206                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9207
9208         /*
9209          * WaIncreaseL3CreditsForVLVB0:vlv
9210          * This is the hardware default actually.
9211          */
9212         I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
9213
9214         /*
9215          * WaDisableVLVClockGating_VBIIssue:vlv
9216          * Disable clock gating on th GCFG unit to prevent a delay
9217          * in the reporting of vblank events.
9218          */
9219         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
9220 }
9221
9222 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
9223 {
9224         /* WaVSRefCountFullforceMissDisable:chv */
9225         /* WaDSRefCountFullforceMissDisable:chv */
9226         I915_WRITE(GEN7_FF_THREAD_MODE,
9227                    I915_READ(GEN7_FF_THREAD_MODE) &
9228                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
9229
9230         /* WaDisableSemaphoreAndSyncFlipWait:chv */
9231         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
9232                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
9233
9234         /* WaDisableCSUnitClockGating:chv */
9235         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
9236                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
9237
9238         /* WaDisableSDEUnitClockGating:chv */
9239         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
9240                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
9241
9242         /*
9243          * WaProgramL3SqcReg1Default:chv
9244          * See gfxspecs/Related Documents/Performance Guide/
9245          * LSQC Setting Recommendations.
9246          */
9247         gen8_set_l3sqc_credits(dev_priv, 38, 2);
9248
9249         /*
9250          * GTT cache may not work with big pages, so if those
9251          * are ever enabled GTT cache may need to be disabled.
9252          */
9253         I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
9254 }
9255
9256 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
9257 {
9258         uint32_t dspclk_gate;
9259
9260         I915_WRITE(RENCLK_GATE_D1, 0);
9261         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
9262                    GS_UNIT_CLOCK_GATE_DISABLE |
9263                    CL_UNIT_CLOCK_GATE_DISABLE);
9264         I915_WRITE(RAMCLK_GATE_D, 0);
9265         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
9266                 OVRUNIT_CLOCK_GATE_DISABLE |
9267                 OVCUNIT_CLOCK_GATE_DISABLE;
9268         if (IS_GM45(dev_priv))
9269                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
9270         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
9271
9272         /* WaDisableRenderCachePipelinedFlush */
9273         I915_WRITE(CACHE_MODE_0,
9274                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
9275
9276         /* WaDisable_RenderCache_OperationalFlush:g4x */
9277         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9278
9279         g4x_disable_trickle_feed(dev_priv);
9280 }
9281
9282 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
9283 {
9284         I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
9285         I915_WRITE(RENCLK_GATE_D2, 0);
9286         I915_WRITE(DSPCLK_GATE_D, 0);
9287         I915_WRITE(RAMCLK_GATE_D, 0);
9288         I915_WRITE16(DEUC, 0);
9289         I915_WRITE(MI_ARB_STATE,
9290                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9291
9292         /* WaDisable_RenderCache_OperationalFlush:gen4 */
9293         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9294 }
9295
9296 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
9297 {
9298         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
9299                    I965_RCC_CLOCK_GATE_DISABLE |
9300                    I965_RCPB_CLOCK_GATE_DISABLE |
9301                    I965_ISC_CLOCK_GATE_DISABLE |
9302                    I965_FBC_CLOCK_GATE_DISABLE);
9303         I915_WRITE(RENCLK_GATE_D2, 0);
9304         I915_WRITE(MI_ARB_STATE,
9305                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9306
9307         /* WaDisable_RenderCache_OperationalFlush:gen4 */
9308         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9309 }
9310
9311 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
9312 {
9313         u32 dstate = I915_READ(D_STATE);
9314
9315         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
9316                 DSTATE_DOT_CLOCK_GATING;
9317         I915_WRITE(D_STATE, dstate);
9318
9319         if (IS_PINEVIEW(dev_priv))
9320                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
9321
9322         /* IIR "flip pending" means done if this bit is set */
9323         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
9324
9325         /* interrupts should cause a wake up from C3 */
9326         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
9327
9328         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
9329         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
9330
9331         I915_WRITE(MI_ARB_STATE,
9332                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9333 }
9334
9335 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
9336 {
9337         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
9338
9339         /* interrupts should cause a wake up from C3 */
9340         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
9341                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
9342
9343         I915_WRITE(MEM_MODE,
9344                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
9345 }
9346
9347 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
9348 {
9349         I915_WRITE(MEM_MODE,
9350                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
9351                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
9352 }
9353
9354 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
9355 {
9356         dev_priv->display.init_clock_gating(dev_priv);
9357 }
9358
9359 void intel_suspend_hw(struct drm_i915_private *dev_priv)
9360 {
9361         if (HAS_PCH_LPT(dev_priv))
9362                 lpt_suspend_hw(dev_priv);
9363 }
9364
9365 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
9366 {
9367         DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n");
9368 }
9369
9370 /**
9371  * intel_init_clock_gating_hooks - setup the clock gating hooks
9372  * @dev_priv: device private
9373  *
9374  * Setup the hooks that configure which clocks of a given platform can be
9375  * gated and also apply various GT and display specific workarounds for these
9376  * platforms. Note that some GT specific workarounds are applied separately
9377  * when GPU contexts or batchbuffers start their execution.
9378  */
9379 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
9380 {
9381         if (IS_ICELAKE(dev_priv))
9382                 dev_priv->display.init_clock_gating = icl_init_clock_gating;
9383         else if (IS_CANNONLAKE(dev_priv))
9384                 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
9385         else if (IS_COFFEELAKE(dev_priv))
9386                 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
9387         else if (IS_SKYLAKE(dev_priv))
9388                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
9389         else if (IS_KABYLAKE(dev_priv))
9390                 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
9391         else if (IS_BROXTON(dev_priv))
9392                 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
9393         else if (IS_GEMINILAKE(dev_priv))
9394                 dev_priv->display.init_clock_gating = glk_init_clock_gating;
9395         else if (IS_BROADWELL(dev_priv))
9396                 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
9397         else if (IS_CHERRYVIEW(dev_priv))
9398                 dev_priv->display.init_clock_gating = chv_init_clock_gating;
9399         else if (IS_HASWELL(dev_priv))
9400                 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
9401         else if (IS_IVYBRIDGE(dev_priv))
9402                 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
9403         else if (IS_VALLEYVIEW(dev_priv))
9404                 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
9405         else if (IS_GEN6(dev_priv))
9406                 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
9407         else if (IS_GEN5(dev_priv))
9408                 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
9409         else if (IS_G4X(dev_priv))
9410                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
9411         else if (IS_I965GM(dev_priv))
9412                 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
9413         else if (IS_I965G(dev_priv))
9414                 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
9415         else if (IS_GEN3(dev_priv))
9416                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
9417         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
9418                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
9419         else if (IS_GEN2(dev_priv))
9420                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
9421         else {
9422                 MISSING_CASE(INTEL_DEVID(dev_priv));
9423                 dev_priv->display.init_clock_gating = nop_init_clock_gating;
9424         }
9425 }
9426
9427 /* Set up chip specific power management-related functions */
9428 void intel_init_pm(struct drm_i915_private *dev_priv)
9429 {
9430         intel_fbc_init(dev_priv);
9431
9432         /* For cxsr */
9433         if (IS_PINEVIEW(dev_priv))
9434                 i915_pineview_get_mem_freq(dev_priv);
9435         else if (IS_GEN5(dev_priv))
9436                 i915_ironlake_get_mem_freq(dev_priv);
9437
9438         /* For FIFO watermark updates */
9439         if (INTEL_GEN(dev_priv) >= 9) {
9440                 skl_setup_wm_latency(dev_priv);
9441                 dev_priv->display.initial_watermarks = skl_initial_wm;
9442                 dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm;
9443                 dev_priv->display.compute_global_watermarks = skl_compute_wm;
9444         } else if (HAS_PCH_SPLIT(dev_priv)) {
9445                 ilk_setup_wm_latency(dev_priv);
9446
9447                 if ((IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[1] &&
9448                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
9449                     (!IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[0] &&
9450                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
9451                         dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
9452                         dev_priv->display.compute_intermediate_wm =
9453                                 ilk_compute_intermediate_wm;
9454                         dev_priv->display.initial_watermarks =
9455                                 ilk_initial_watermarks;
9456                         dev_priv->display.optimize_watermarks =
9457                                 ilk_optimize_watermarks;
9458                 } else {
9459                         DRM_DEBUG_KMS("Failed to read display plane latency. "
9460                                       "Disable CxSR\n");
9461                 }
9462         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9463                 vlv_setup_wm_latency(dev_priv);
9464                 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
9465                 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
9466                 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
9467                 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
9468                 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
9469         } else if (IS_G4X(dev_priv)) {
9470                 g4x_setup_wm_latency(dev_priv);
9471                 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
9472                 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
9473                 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
9474                 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
9475         } else if (IS_PINEVIEW(dev_priv)) {
9476                 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
9477                                             dev_priv->is_ddr3,
9478                                             dev_priv->fsb_freq,
9479                                             dev_priv->mem_freq)) {
9480                         DRM_INFO("failed to find known CxSR latency "
9481                                  "(found ddr%s fsb freq %d, mem freq %d), "
9482                                  "disabling CxSR\n",
9483                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
9484                                  dev_priv->fsb_freq, dev_priv->mem_freq);
9485                         /* Disable CxSR and never update its watermark again */
9486                         intel_set_memory_cxsr(dev_priv, false);
9487                         dev_priv->display.update_wm = NULL;
9488                 } else
9489                         dev_priv->display.update_wm = pineview_update_wm;
9490         } else if (IS_GEN4(dev_priv)) {
9491                 dev_priv->display.update_wm = i965_update_wm;
9492         } else if (IS_GEN3(dev_priv)) {
9493                 dev_priv->display.update_wm = i9xx_update_wm;
9494                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
9495         } else if (IS_GEN2(dev_priv)) {
9496                 if (INTEL_INFO(dev_priv)->num_pipes == 1) {
9497                         dev_priv->display.update_wm = i845_update_wm;
9498                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
9499                 } else {
9500                         dev_priv->display.update_wm = i9xx_update_wm;
9501                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
9502                 }
9503         } else {
9504                 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
9505         }
9506 }
9507
9508 static inline int gen6_check_mailbox_status(struct drm_i915_private *dev_priv)
9509 {
9510         uint32_t flags =
9511                 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
9512
9513         switch (flags) {
9514         case GEN6_PCODE_SUCCESS:
9515                 return 0;
9516         case GEN6_PCODE_UNIMPLEMENTED_CMD:
9517                 return -ENODEV;
9518         case GEN6_PCODE_ILLEGAL_CMD:
9519                 return -ENXIO;
9520         case GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9521         case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9522                 return -EOVERFLOW;
9523         case GEN6_PCODE_TIMEOUT:
9524                 return -ETIMEDOUT;
9525         default:
9526                 MISSING_CASE(flags);
9527                 return 0;
9528         }
9529 }
9530
9531 static inline int gen7_check_mailbox_status(struct drm_i915_private *dev_priv)
9532 {
9533         uint32_t flags =
9534                 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
9535
9536         switch (flags) {
9537         case GEN6_PCODE_SUCCESS:
9538                 return 0;
9539         case GEN6_PCODE_ILLEGAL_CMD:
9540                 return -ENXIO;
9541         case GEN7_PCODE_TIMEOUT:
9542                 return -ETIMEDOUT;
9543         case GEN7_PCODE_ILLEGAL_DATA:
9544                 return -EINVAL;
9545         case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9546                 return -EOVERFLOW;
9547         default:
9548                 MISSING_CASE(flags);
9549                 return 0;
9550         }
9551 }
9552
9553 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
9554 {
9555         int status;
9556
9557         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9558
9559         /* GEN6_PCODE_* are outside of the forcewake domain, we can
9560          * use te fw I915_READ variants to reduce the amount of work
9561          * required when reading/writing.
9562          */
9563
9564         if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
9565                 DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps\n",
9566                                  mbox, __builtin_return_address(0));
9567                 return -EAGAIN;
9568         }
9569
9570         I915_WRITE_FW(GEN6_PCODE_DATA, *val);
9571         I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
9572         I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
9573
9574         if (__intel_wait_for_register_fw(dev_priv,
9575                                          GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
9576                                          500, 0, NULL)) {
9577                 DRM_ERROR("timeout waiting for pcode read (from mbox %x) to finish for %ps\n",
9578                           mbox, __builtin_return_address(0));
9579                 return -ETIMEDOUT;
9580         }
9581
9582         *val = I915_READ_FW(GEN6_PCODE_DATA);
9583         I915_WRITE_FW(GEN6_PCODE_DATA, 0);
9584
9585         if (INTEL_GEN(dev_priv) > 6)
9586                 status = gen7_check_mailbox_status(dev_priv);
9587         else
9588                 status = gen6_check_mailbox_status(dev_priv);
9589
9590         if (status) {
9591                 DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps: %d\n",
9592                                  mbox, __builtin_return_address(0), status);
9593                 return status;
9594         }
9595
9596         return 0;
9597 }
9598
9599 int sandybridge_pcode_write_timeout(struct drm_i915_private *dev_priv,
9600                                     u32 mbox, u32 val,
9601                                     int fast_timeout_us, int slow_timeout_ms)
9602 {
9603         int status;
9604
9605         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9606
9607         /* GEN6_PCODE_* are outside of the forcewake domain, we can
9608          * use te fw I915_READ variants to reduce the amount of work
9609          * required when reading/writing.
9610          */
9611
9612         if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
9613                 DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps\n",
9614                                  val, mbox, __builtin_return_address(0));
9615                 return -EAGAIN;
9616         }
9617
9618         I915_WRITE_FW(GEN6_PCODE_DATA, val);
9619         I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
9620         I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
9621
9622         if (__intel_wait_for_register_fw(dev_priv,
9623                                          GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
9624                                          fast_timeout_us, slow_timeout_ms,
9625                                          NULL)) {
9626                 DRM_ERROR("timeout waiting for pcode write of 0x%08x to mbox %x to finish for %ps\n",
9627                           val, mbox, __builtin_return_address(0));
9628                 return -ETIMEDOUT;
9629         }
9630
9631         I915_WRITE_FW(GEN6_PCODE_DATA, 0);
9632
9633         if (INTEL_GEN(dev_priv) > 6)
9634                 status = gen7_check_mailbox_status(dev_priv);
9635         else
9636                 status = gen6_check_mailbox_status(dev_priv);
9637
9638         if (status) {
9639                 DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps: %d\n",
9640                                  val, mbox, __builtin_return_address(0), status);
9641                 return status;
9642         }
9643
9644         return 0;
9645 }
9646
9647 static bool skl_pcode_try_request(struct drm_i915_private *dev_priv, u32 mbox,
9648                                   u32 request, u32 reply_mask, u32 reply,
9649                                   u32 *status)
9650 {
9651         u32 val = request;
9652
9653         *status = sandybridge_pcode_read(dev_priv, mbox, &val);
9654
9655         return *status || ((val & reply_mask) == reply);
9656 }
9657
9658 /**
9659  * skl_pcode_request - send PCODE request until acknowledgment
9660  * @dev_priv: device private
9661  * @mbox: PCODE mailbox ID the request is targeted for
9662  * @request: request ID
9663  * @reply_mask: mask used to check for request acknowledgment
9664  * @reply: value used to check for request acknowledgment
9665  * @timeout_base_ms: timeout for polling with preemption enabled
9666  *
9667  * Keep resending the @request to @mbox until PCODE acknowledges it, PCODE
9668  * reports an error or an overall timeout of @timeout_base_ms+50 ms expires.
9669  * The request is acknowledged once the PCODE reply dword equals @reply after
9670  * applying @reply_mask. Polling is first attempted with preemption enabled
9671  * for @timeout_base_ms and if this times out for another 50 ms with
9672  * preemption disabled.
9673  *
9674  * Returns 0 on success, %-ETIMEDOUT in case of a timeout, <0 in case of some
9675  * other error as reported by PCODE.
9676  */
9677 int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request,
9678                       u32 reply_mask, u32 reply, int timeout_base_ms)
9679 {
9680         u32 status;
9681         int ret;
9682
9683         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9684
9685 #define COND skl_pcode_try_request(dev_priv, mbox, request, reply_mask, reply, \
9686                                    &status)
9687
9688         /*
9689          * Prime the PCODE by doing a request first. Normally it guarantees
9690          * that a subsequent request, at most @timeout_base_ms later, succeeds.
9691          * _wait_for() doesn't guarantee when its passed condition is evaluated
9692          * first, so send the first request explicitly.
9693          */
9694         if (COND) {
9695                 ret = 0;
9696                 goto out;
9697         }
9698         ret = _wait_for(COND, timeout_base_ms * 1000, 10, 10);
9699         if (!ret)
9700                 goto out;
9701
9702         /*
9703          * The above can time out if the number of requests was low (2 in the
9704          * worst case) _and_ PCODE was busy for some reason even after a
9705          * (queued) request and @timeout_base_ms delay. As a workaround retry
9706          * the poll with preemption disabled to maximize the number of
9707          * requests. Increase the timeout from @timeout_base_ms to 50ms to
9708          * account for interrupts that could reduce the number of these
9709          * requests, and for any quirks of the PCODE firmware that delays
9710          * the request completion.
9711          */
9712         DRM_DEBUG_KMS("PCODE timeout, retrying with preemption disabled\n");
9713         WARN_ON_ONCE(timeout_base_ms > 3);
9714         preempt_disable();
9715         ret = wait_for_atomic(COND, 50);
9716         preempt_enable();
9717
9718 out:
9719         return ret ? ret : status;
9720 #undef COND
9721 }
9722
9723 static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
9724 {
9725         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9726
9727         /*
9728          * N = val - 0xb7
9729          * Slow = Fast = GPLL ref * N
9730          */
9731         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
9732 }
9733
9734 static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
9735 {
9736         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9737
9738         return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
9739 }
9740
9741 static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
9742 {
9743         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9744
9745         /*
9746          * N = val / 2
9747          * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
9748          */
9749         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
9750 }
9751
9752 static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
9753 {
9754         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9755
9756         /* CHV needs even values */
9757         return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
9758 }
9759
9760 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
9761 {
9762         if (INTEL_GEN(dev_priv) >= 9)
9763                 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
9764                                          GEN9_FREQ_SCALER);
9765         else if (IS_CHERRYVIEW(dev_priv))
9766                 return chv_gpu_freq(dev_priv, val);
9767         else if (IS_VALLEYVIEW(dev_priv))
9768                 return byt_gpu_freq(dev_priv, val);
9769         else
9770                 return val * GT_FREQUENCY_MULTIPLIER;
9771 }
9772
9773 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
9774 {
9775         if (INTEL_GEN(dev_priv) >= 9)
9776                 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
9777                                          GT_FREQUENCY_MULTIPLIER);
9778         else if (IS_CHERRYVIEW(dev_priv))
9779                 return chv_freq_opcode(dev_priv, val);
9780         else if (IS_VALLEYVIEW(dev_priv))
9781                 return byt_freq_opcode(dev_priv, val);
9782         else
9783                 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
9784 }
9785
9786 void intel_pm_setup(struct drm_i915_private *dev_priv)
9787 {
9788         mutex_init(&dev_priv->pcu_lock);
9789         mutex_init(&dev_priv->gt_pm.rps.power.mutex);
9790
9791         atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
9792
9793         dev_priv->runtime_pm.suspended = false;
9794         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
9795 }
9796
9797 static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
9798                              const i915_reg_t reg)
9799 {
9800         u32 lower, upper, tmp;
9801         int loop = 2;
9802
9803         /*
9804          * The register accessed do not need forcewake. We borrow
9805          * uncore lock to prevent concurrent access to range reg.
9806          */
9807         lockdep_assert_held(&dev_priv->uncore.lock);
9808
9809         /*
9810          * vlv and chv residency counters are 40 bits in width.
9811          * With a control bit, we can choose between upper or lower
9812          * 32bit window into this counter.
9813          *
9814          * Although we always use the counter in high-range mode elsewhere,
9815          * userspace may attempt to read the value before rc6 is initialised,
9816          * before we have set the default VLV_COUNTER_CONTROL value. So always
9817          * set the high bit to be safe.
9818          */
9819         I915_WRITE_FW(VLV_COUNTER_CONTROL,
9820                       _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9821         upper = I915_READ_FW(reg);
9822         do {
9823                 tmp = upper;
9824
9825                 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9826                               _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
9827                 lower = I915_READ_FW(reg);
9828
9829                 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9830                               _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9831                 upper = I915_READ_FW(reg);
9832         } while (upper != tmp && --loop);
9833
9834         /*
9835          * Everywhere else we always use VLV_COUNTER_CONTROL with the
9836          * VLV_COUNT_RANGE_HIGH bit set - so it is safe to leave it set
9837          * now.
9838          */
9839
9840         return lower | (u64)upper << 8;
9841 }
9842
9843 u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
9844                            const i915_reg_t reg)
9845 {
9846         u64 time_hw, prev_hw, overflow_hw;
9847         unsigned int fw_domains;
9848         unsigned long flags;
9849         unsigned int i;
9850         u32 mul, div;
9851
9852         if (!HAS_RC6(dev_priv))
9853                 return 0;
9854
9855         /*
9856          * Store previous hw counter values for counter wrap-around handling.
9857          *
9858          * There are only four interesting registers and they live next to each
9859          * other so we can use the relative address, compared to the smallest
9860          * one as the index into driver storage.
9861          */
9862         i = (i915_mmio_reg_offset(reg) -
9863              i915_mmio_reg_offset(GEN6_GT_GFX_RC6_LOCKED)) / sizeof(u32);
9864         if (WARN_ON_ONCE(i >= ARRAY_SIZE(dev_priv->gt_pm.rc6.cur_residency)))
9865                 return 0;
9866
9867         fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
9868
9869         spin_lock_irqsave(&dev_priv->uncore.lock, flags);
9870         intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
9871
9872         /* On VLV and CHV, residency time is in CZ units rather than 1.28us */
9873         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9874                 mul = 1000000;
9875                 div = dev_priv->czclk_freq;
9876                 overflow_hw = BIT_ULL(40);
9877                 time_hw = vlv_residency_raw(dev_priv, reg);
9878         } else {
9879                 /* 833.33ns units on Gen9LP, 1.28us elsewhere. */
9880                 if (IS_GEN9_LP(dev_priv)) {
9881                         mul = 10000;
9882                         div = 12;
9883                 } else {
9884                         mul = 1280;
9885                         div = 1;
9886                 }
9887
9888                 overflow_hw = BIT_ULL(32);
9889                 time_hw = I915_READ_FW(reg);
9890         }
9891
9892         /*
9893          * Counter wrap handling.
9894          *
9895          * But relying on a sufficient frequency of queries otherwise counters
9896          * can still wrap.
9897          */
9898         prev_hw = dev_priv->gt_pm.rc6.prev_hw_residency[i];
9899         dev_priv->gt_pm.rc6.prev_hw_residency[i] = time_hw;
9900
9901         /* RC6 delta from last sample. */
9902         if (time_hw >= prev_hw)
9903                 time_hw -= prev_hw;
9904         else
9905                 time_hw += overflow_hw - prev_hw;
9906
9907         /* Add delta to RC6 extended raw driver copy. */
9908         time_hw += dev_priv->gt_pm.rc6.cur_residency[i];
9909         dev_priv->gt_pm.rc6.cur_residency[i] = time_hw;
9910
9911         intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
9912         spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
9913
9914         return mul_u64_u32_div(time_hw, mul, div);
9915 }
9916
9917 u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat)
9918 {
9919         u32 cagf;
9920
9921         if (INTEL_GEN(dev_priv) >= 9)
9922                 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
9923         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
9924                 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
9925         else
9926                 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
9927
9928         return  cagf;
9929 }