drivers/gpu/drm/i915/intel_runtime_pm.c

   1 /*
   2  * Copyright © 2012-2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *    Daniel Vetter <daniel.vetter@ffwll.ch>
  26  *
  27  */
  28
  29 #include <linux/pm_runtime.h>
  30 #include <linux/vgaarb.h>
  31
  32 #include "i915_drv.h"
  33 #include "intel_drv.h"
  34
  35 /**
  36  * DOC: runtime pm
  37  *
  38  * The i915 driver supports dynamic enabling and disabling of entire hardware
  39  * blocks at runtime. This is especially important on the display side where
  40  * software is supposed to control many power gates manually on recent hardware,
  41  * since on the GT side a lot of the power management is done by the hardware.
  42  * But even there some manual control at the device level is required.
  43  *
  44  * Since i915 supports a diverse set of platforms with a unified codebase and
  45  * hardware engineers just love to shuffle functionality around between power
  46  * domains there's a sizeable amount of indirection required. This file provides
  47  * generic functions to the driver for grabbing and releasing references for
  48  * abstract power domains. It then maps those to the actual power wells
  49  * present for a given platform.
  50  */
  51
  52 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
  53                                          enum i915_power_well_id power_well_id);
  54
  55 static struct i915_power_well *
  56 lookup_power_well(struct drm_i915_private *dev_priv,
  57                   enum i915_power_well_id power_well_id);
  58
  59 const char *
  60 intel_display_power_domain_str(enum intel_display_power_domain domain)
  61 {
  62         switch (domain) {
  63         case POWER_DOMAIN_PIPE_A:
  64                 return "PIPE_A";
  65         case POWER_DOMAIN_PIPE_B:
  66                 return "PIPE_B";
  67         case POWER_DOMAIN_PIPE_C:
  68                 return "PIPE_C";
  69         case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
  70                 return "PIPE_A_PANEL_FITTER";
  71         case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
  72                 return "PIPE_B_PANEL_FITTER";
  73         case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
  74                 return "PIPE_C_PANEL_FITTER";
  75         case POWER_DOMAIN_TRANSCODER_A:
  76                 return "TRANSCODER_A";
  77         case POWER_DOMAIN_TRANSCODER_B:
  78                 return "TRANSCODER_B";
  79         case POWER_DOMAIN_TRANSCODER_C:
  80                 return "TRANSCODER_C";
  81         case POWER_DOMAIN_TRANSCODER_EDP:
  82                 return "TRANSCODER_EDP";
  83         case POWER_DOMAIN_TRANSCODER_DSI_A:
  84                 return "TRANSCODER_DSI_A";
  85         case POWER_DOMAIN_TRANSCODER_DSI_C:
  86                 return "TRANSCODER_DSI_C";
  87         case POWER_DOMAIN_PORT_DDI_A_LANES:
  88                 return "PORT_DDI_A_LANES";
  89         case POWER_DOMAIN_PORT_DDI_B_LANES:
  90                 return "PORT_DDI_B_LANES";
  91         case POWER_DOMAIN_PORT_DDI_C_LANES:
  92                 return "PORT_DDI_C_LANES";
  93         case POWER_DOMAIN_PORT_DDI_D_LANES:
  94                 return "PORT_DDI_D_LANES";
  95         case POWER_DOMAIN_PORT_DDI_E_LANES:
  96                 return "PORT_DDI_E_LANES";
  97         case POWER_DOMAIN_PORT_DDI_A_IO:
  98                 return "PORT_DDI_A_IO";
  99         case POWER_DOMAIN_PORT_DDI_B_IO:
 100                 return "PORT_DDI_B_IO";
 101         case POWER_DOMAIN_PORT_DDI_C_IO:
 102                 return "PORT_DDI_C_IO";
 103         case POWER_DOMAIN_PORT_DDI_D_IO:
 104                 return "PORT_DDI_D_IO";
 105         case POWER_DOMAIN_PORT_DDI_E_IO:
 106                 return "PORT_DDI_E_IO";
 107         case POWER_DOMAIN_PORT_DSI:
 108                 return "PORT_DSI";
 109         case POWER_DOMAIN_PORT_CRT:
 110                 return "PORT_CRT";
 111         case POWER_DOMAIN_PORT_OTHER:
 112                 return "PORT_OTHER";
 113         case POWER_DOMAIN_VGA:
 114                 return "VGA";
 115         case POWER_DOMAIN_AUDIO:
 116                 return "AUDIO";
 117         case POWER_DOMAIN_PLLS:
 118                 return "PLLS";
 119         case POWER_DOMAIN_AUX_A:
 120                 return "AUX_A";
 121         case POWER_DOMAIN_AUX_B:
 122                 return "AUX_B";
 123         case POWER_DOMAIN_AUX_C:
 124                 return "AUX_C";
 125         case POWER_DOMAIN_AUX_D:
 126                 return "AUX_D";
 127         case POWER_DOMAIN_GMBUS:
 128                 return "GMBUS";
 129         case POWER_DOMAIN_INIT:
 130                 return "INIT";
 131         case POWER_DOMAIN_MODESET:
 132                 return "MODESET";
 133         default:
 134                 MISSING_CASE(domain);
 135                 return "?";
 136         }
 137 }
 138
 139 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
 140                                     struct i915_power_well *power_well)
 141 {
 142         DRM_DEBUG_KMS("enabling %s\n", power_well->name);
 143         power_well->ops->enable(dev_priv, power_well);
 144         power_well->hw_enabled = true;
 145 }
 146
 147 static void intel_power_well_disable(struct drm_i915_private *dev_priv,
 148                                      struct i915_power_well *power_well)
 149 {
 150         DRM_DEBUG_KMS("disabling %s\n", power_well->name);
 151         power_well->hw_enabled = false;
 152         power_well->ops->disable(dev_priv, power_well);
 153 }
 154
 155 static void intel_power_well_get(struct drm_i915_private *dev_priv,
 156                                  struct i915_power_well *power_well)
 157 {
 158         if (!power_well->count++)
 159                 intel_power_well_enable(dev_priv, power_well);
 160 }
 161
 162 static void intel_power_well_put(struct drm_i915_private *dev_priv,
 163                                  struct i915_power_well *power_well)
 164 {
 165         WARN(!power_well->count, "Use count on power well %s is already zero",
 166              power_well->name);
 167
 168         if (!--power_well->count)
 169                 intel_power_well_disable(dev_priv, power_well);
 170 }
 171
 172 /**
 173  * __intel_display_power_is_enabled - unlocked check for a power domain
 174  * @dev_priv: i915 device instance
 175  * @domain: power domain to check
 176  *
 177  * This is the unlocked version of intel_display_power_is_enabled() and should
 178  * only be used from error capture and recovery code where deadlocks are
 179  * possible.
 180  *
 181  * Returns:
 182  * True when the power domain is enabled, false otherwise.
 183  */
 184 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 185                                       enum intel_display_power_domain domain)
 186 {
 187         struct i915_power_well *power_well;
 188         bool is_enabled;
 189
 190         if (dev_priv->pm.suspended)
 191                 return false;
 192
 193         is_enabled = true;
 194
 195         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain)) {
 196                 if (power_well->always_on)
 197                         continue;
 198
 199                 if (!power_well->hw_enabled) {
 200                         is_enabled = false;
 201                         break;
 202                 }
 203         }
 204
 205         return is_enabled;
 206 }
 207
 208 /**
 209  * intel_display_power_is_enabled - check for a power domain
 210  * @dev_priv: i915 device instance
 211  * @domain: power domain to check
 212  *
 213  * This function can be used to check the hw power domain state. It is mostly
 214  * used in hardware state readout functions. Everywhere else code should rely
 215  * upon explicit power domain reference counting to ensure that the hardware
 216  * block is powered up before accessing it.
 217  *
 218  * Callers must hold the relevant modesetting locks to ensure that concurrent
 219  * threads can't disable the power well while the caller tries to read a few
 220  * registers.
 221  *
 222  * Returns:
 223  * True when the power domain is enabled, false otherwise.
 224  */
 225 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 226                                     enum intel_display_power_domain domain)
 227 {
 228         struct i915_power_domains *power_domains;
 229         bool ret;
 230
 231         power_domains = &dev_priv->power_domains;
 232
 233         mutex_lock(&power_domains->lock);
 234         ret = __intel_display_power_is_enabled(dev_priv, domain);
 235         mutex_unlock(&power_domains->lock);
 236
 237         return ret;
 238 }
 239
 240 /**
 241  * intel_display_set_init_power - set the initial power domain state
 242  * @dev_priv: i915 device instance
 243  * @enable: whether to enable or disable the initial power domain state
 244  *
 245  * For simplicity our driver load/unload and system suspend/resume code assumes
 246  * that all power domains are always enabled. This functions controls the state
 247  * of this little hack. While the initial power domain state is enabled runtime
 248  * pm is effectively disabled.
 249  */
 250 void intel_display_set_init_power(struct drm_i915_private *dev_priv,
 251                                   bool enable)
 252 {
 253         if (dev_priv->power_domains.init_power_on == enable)
 254                 return;
 255
 256         if (enable)
 257                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
 258         else
 259                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 260
 261         dev_priv->power_domains.init_power_on = enable;
 262 }
 263
 264 /*
 265  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 266  * when not needed anymore. We have 4 registers that can request the power well
 267  * to be enabled, and it will only be disabled if none of the registers is
 268  * requesting it to be enabled.
 269  */
 270 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
 271                                        u8 irq_pipe_mask, bool has_vga)
 272 {
 273         struct pci_dev *pdev = dev_priv->drm.pdev;
 274
 275         /*
 276          * After we re-enable the power well, if we touch VGA register 0x3d5
 277          * we'll get unclaimed register interrupts. This stops after we write
 278          * anything to the VGA MSR register. The vgacon module uses this
 279          * register all the time, so if we unbind our driver and, as a
 280          * consequence, bind vgacon, we'll get stuck in an infinite loop at
 281          * console_unlock(). So make here we touch the VGA MSR register, making
 282          * sure vgacon can keep working normally without triggering interrupts
 283          * and error messages.
 284          */
 285         if (has_vga) {
 286                 vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 287                 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 288                 vga_put(pdev, VGA_RSRC_LEGACY_IO);
 289         }
 290
 291         if (irq_pipe_mask)
 292                 gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 293 }
 294
 295 static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
 296                                        u8 irq_pipe_mask)
 297 {
 298         if (irq_pipe_mask)
 299                 gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 300 }
 301
 302
 303 static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
 304                                            struct i915_power_well *power_well)
 305 {
 306         enum i915_power_well_id id = power_well->id;
 307
 308         /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
 309         WARN_ON(intel_wait_for_register(dev_priv,
 310                                         HSW_PWR_WELL_CTL_DRIVER(id),
 311                                         HSW_PWR_WELL_CTL_STATE(id),
 312                                         HSW_PWR_WELL_CTL_STATE(id),
 313                                         1));
 314 }
 315
 316 static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
 317                                      enum i915_power_well_id id)
 318 {
 319         u32 req_mask = HSW_PWR_WELL_CTL_REQ(id);
 320         u32 ret;
 321
 322         ret = I915_READ(HSW_PWR_WELL_CTL_BIOS(id)) & req_mask ? 1 : 0;
 323         ret |= I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & req_mask ? 2 : 0;
 324         ret |= I915_READ(HSW_PWR_WELL_CTL_KVMR) & req_mask ? 4 : 0;
 325         ret |= I915_READ(HSW_PWR_WELL_CTL_DEBUG(id)) & req_mask ? 8 : 0;
 326
 327         return ret;
 328 }
 329
 330 static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
 331                                             struct i915_power_well *power_well)
 332 {
 333         enum i915_power_well_id id = power_well->id;
 334         bool disabled;
 335         u32 reqs;
 336
 337         /*
 338          * Bspec doesn't require waiting for PWs to get disabled, but still do
 339          * this for paranoia. The known cases where a PW will be forced on:
 340          * - a KVMR request on any power well via the KVMR request register
 341          * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
 342          *   DEBUG request registers
 343          * Skip the wait in case any of the request bits are set and print a
 344          * diagnostic message.
 345          */
 346         wait_for((disabled = !(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
 347                                HSW_PWR_WELL_CTL_STATE(id))) ||
 348                  (reqs = hsw_power_well_requesters(dev_priv, id)), 1);
 349         if (disabled)
 350                 return;
 351
 352         DRM_DEBUG_KMS("%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
 353                       power_well->name,
 354                       !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 355 }
 356
 357 static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
 358                                            enum skl_power_gate pg)
 359 {
 360         /* Timeout 5us for PG#0, for other PGs 1us */
 361         WARN_ON(intel_wait_for_register(dev_priv, SKL_FUSE_STATUS,
 362                                         SKL_FUSE_PG_DIST_STATUS(pg),
 363                                         SKL_FUSE_PG_DIST_STATUS(pg), 1));
 364 }
 365
 366 static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 367                                   struct i915_power_well *power_well)
 368 {
 369         enum i915_power_well_id id = power_well->id;
 370         bool wait_fuses = power_well->hsw.has_fuses;
 371         enum skl_power_gate uninitialized_var(pg);
 372         u32 val;
 373
 374         if (wait_fuses) {
 375                 pg = SKL_PW_TO_PG(id);
 376                 /*
 377                  * For PW1 we have to wait both for the PW0/PG0 fuse state
 378                  * before enabling the power well and PW1/PG1's own fuse
 379                  * state after the enabling. For all other power wells with
 380                  * fuses we only have to wait for that PW/PG's fuse state
 381                  * after the enabling.
 382                  */
 383                 if (pg == SKL_PG1)
 384                         gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 385         }
 386
 387         val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 388         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), val | HSW_PWR_WELL_CTL_REQ(id));
 389         hsw_wait_for_power_well_enable(dev_priv, power_well);
 390
 391         if (wait_fuses)
 392                 gen9_wait_for_power_well_fuses(dev_priv, pg);
 393
 394         hsw_power_well_post_enable(dev_priv, power_well->hsw.irq_pipe_mask,
 395                                    power_well->hsw.has_vga);
 396 }
 397
 398 static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 399                                    struct i915_power_well *power_well)
 400 {
 401         enum i915_power_well_id id = power_well->id;
 402         u32 val;
 403
 404         hsw_power_well_pre_disable(dev_priv, power_well->hsw.irq_pipe_mask);
 405
 406         val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 407         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id),
 408                    val & ~HSW_PWR_WELL_CTL_REQ(id));
 409         hsw_wait_for_power_well_disable(dev_priv, power_well);
 410 }
 411
 412 /*
 413  * We should only use the power well if we explicitly asked the hardware to
 414  * enable it, so check if it's enabled and also check if we've requested it to
 415  * be enabled.
 416  */
 417 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
 418                                    struct i915_power_well *power_well)
 419 {
 420         enum i915_power_well_id id = power_well->id;
 421         u32 mask = HSW_PWR_WELL_CTL_REQ(id) | HSW_PWR_WELL_CTL_STATE(id);
 422
 423         return (I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & mask) == mask;
 424 }
 425
 426 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 427 {
 428         enum i915_power_well_id id = SKL_DISP_PW_2;
 429
 430         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
 431                   "DC9 already programmed to be enabled.\n");
 432         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 433                   "DC5 still not disabled to enable DC9.\n");
 434         WARN_ONCE(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
 435                   HSW_PWR_WELL_CTL_REQ(id),
 436                   "Power well 2 on.\n");
 437         WARN_ONCE(intel_irqs_enabled(dev_priv),
 438                   "Interrupts not disabled yet.\n");
 439
 440          /*
 441           * TODO: check for the following to verify the conditions to enter DC9
 442           * state are satisfied:
 443           * 1] Check relevant display engine registers to verify if mode set
 444           * disable sequence was followed.
 445           * 2] Check if display uninitialize sequence is initialized.
 446           */
 447 }
 448
 449 static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 450 {
 451         WARN_ONCE(intel_irqs_enabled(dev_priv),
 452                   "Interrupts not disabled yet.\n");
 453         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 454                   "DC5 still not disabled.\n");
 455
 456          /*
 457           * TODO: check for the following to verify DC9 state was indeed
 458           * entered before programming to disable it:
 459           * 1] Check relevant display engine registers to verify if mode
 460           *  set disable sequence was followed.
 461           * 2] Check if display uninitialize sequence is initialized.
 462           */
 463 }
 464
 465 static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
 466                                 u32 state)
 467 {
 468         int rewrites = 0;
 469         int rereads = 0;
 470         u32 v;
 471
 472         I915_WRITE(DC_STATE_EN, state);
 473
 474         /* It has been observed that disabling the dc6 state sometimes
 475          * doesn't stick and dmc keeps returning old value. Make sure
 476          * the write really sticks enough times and also force rewrite until
 477          * we are confident that state is exactly what we want.
 478          */
 479         do  {
 480                 v = I915_READ(DC_STATE_EN);
 481
 482                 if (v != state) {
 483                         I915_WRITE(DC_STATE_EN, state);
 484                         rewrites++;
 485                         rereads = 0;
 486                 } else if (rereads++ > 5) {
 487                         break;
 488                 }
 489
 490         } while (rewrites < 100);
 491
 492         if (v != state)
 493                 DRM_ERROR("Writing dc state to 0x%x failed, now 0x%x\n",
 494                           state, v);
 495
 496         /* Most of the times we need one retry, avoid spam */
 497         if (rewrites > 1)
 498                 DRM_DEBUG_KMS("Rewrote dc state to 0x%x %d times\n",
 499                               state, rewrites);
 500 }
 501
 502 static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
 503 {
 504         u32 mask;
 505
 506         mask = DC_STATE_EN_UPTO_DC5;
 507         if (IS_GEN9_LP(dev_priv))
 508                 mask |= DC_STATE_EN_DC9;
 509         else
 510                 mask |= DC_STATE_EN_UPTO_DC6;
 511
 512         return mask;
 513 }
 514
 515 void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 516 {
 517         u32 val;
 518
 519         val = I915_READ(DC_STATE_EN) & gen9_dc_mask(dev_priv);
 520
 521         DRM_DEBUG_KMS("Resetting DC state tracking from %02x to %02x\n",
 522                       dev_priv->csr.dc_state, val);
 523         dev_priv->csr.dc_state = val;
 524 }
 525
 526 static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
 527 {
 528         uint32_t val;
 529         uint32_t mask;
 530
 531         if (WARN_ON_ONCE(state & ~dev_priv->csr.allowed_dc_mask))
 532                 state &= dev_priv->csr.allowed_dc_mask;
 533
 534         val = I915_READ(DC_STATE_EN);
 535         mask = gen9_dc_mask(dev_priv);
 536         DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n",
 537                       val & mask, state);
 538
 539         /* Check if DMC is ignoring our DC state requests */
 540         if ((val & mask) != dev_priv->csr.dc_state)
 541                 DRM_ERROR("DC state mismatch (0x%x -> 0x%x)\n",
 542                           dev_priv->csr.dc_state, val & mask);
 543
 544         val &= ~mask;
 545         val |= state;
 546
 547         gen9_write_dc_state(dev_priv, val);
 548
 549         dev_priv->csr.dc_state = val & mask;
 550 }
 551
 552 void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 553 {
 554         assert_can_enable_dc9(dev_priv);
 555
 556         DRM_DEBUG_KMS("Enabling DC9\n");
 557
 558         intel_power_sequencer_reset(dev_priv);
 559         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
 560 }
 561
 562 void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 563 {
 564         assert_can_disable_dc9(dev_priv);
 565
 566         DRM_DEBUG_KMS("Disabling DC9\n");
 567
 568         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 569
 570         intel_pps_unlock_regs_wa(dev_priv);
 571 }
 572
 573 static void assert_csr_loaded(struct drm_i915_private *dev_priv)
 574 {
 575         WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
 576                   "CSR program storage start is NULL\n");
 577         WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
 578         WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
 579 }
 580
 581 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
 582 {
 583         bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
 584                                         SKL_DISP_PW_2);
 585
 586         WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n");
 587
 588         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
 589                   "DC5 already programmed to be enabled.\n");
 590         assert_rpm_wakelock_held(dev_priv);
 591
 592         assert_csr_loaded(dev_priv);
 593 }
 594
 595 void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 596 {
 597         assert_can_enable_dc5(dev_priv);
 598
 599         DRM_DEBUG_KMS("Enabling DC5\n");
 600
 601         /* Wa Display #1183: skl,kbl,cfl */
 602         if (IS_GEN9_BC(dev_priv))
 603                 I915_WRITE(GEN8_CHICKEN_DCPR_1, I915_READ(GEN8_CHICKEN_DCPR_1) |
 604                            SKL_SELECT_ALTERNATE_DC_EXIT);
 605
 606         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
 607 }
 608
 609 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
 610 {
 611         WARN_ONCE(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
 612                   "Backlight is not disabled.\n");
 613         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
 614                   "DC6 already programmed to be enabled.\n");
 615
 616         assert_csr_loaded(dev_priv);
 617 }
 618
 619 void skl_enable_dc6(struct drm_i915_private *dev_priv)
 620 {
 621         assert_can_enable_dc6(dev_priv);
 622
 623         DRM_DEBUG_KMS("Enabling DC6\n");
 624
 625         /* Wa Display #1183: skl,kbl,cfl */
 626         if (IS_GEN9_BC(dev_priv))
 627                 I915_WRITE(GEN8_CHICKEN_DCPR_1, I915_READ(GEN8_CHICKEN_DCPR_1) |
 628                            SKL_SELECT_ALTERNATE_DC_EXIT);
 629
 630         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 631 }
 632
 633 void skl_disable_dc6(struct drm_i915_private *dev_priv)
 634 {
 635         DRM_DEBUG_KMS("Disabling DC6\n");
 636
 637         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 638 }
 639
 640 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 641                                    struct i915_power_well *power_well)
 642 {
 643         enum i915_power_well_id id = power_well->id;
 644         u32 mask = HSW_PWR_WELL_CTL_REQ(id);
 645         u32 bios_req = I915_READ(HSW_PWR_WELL_CTL_BIOS(id));
 646
 647         /* Take over the request bit if set by BIOS. */
 648         if (bios_req & mask) {
 649                 u32 drv_req = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 650
 651                 if (!(drv_req & mask))
 652                         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), drv_req | mask);
 653                 I915_WRITE(HSW_PWR_WELL_CTL_BIOS(id), bios_req & ~mask);
 654         }
 655 }
 656
 657 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 658                                            struct i915_power_well *power_well)
 659 {
 660         bxt_ddi_phy_init(dev_priv, power_well->bxt.phy);
 661 }
 662
 663 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 664                                             struct i915_power_well *power_well)
 665 {
 666         bxt_ddi_phy_uninit(dev_priv, power_well->bxt.phy);
 667 }
 668
 669 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
 670                                             struct i915_power_well *power_well)
 671 {
 672         return bxt_ddi_phy_is_enabled(dev_priv, power_well->bxt.phy);
 673 }
 674
 675 static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
 676 {
 677         struct i915_power_well *power_well;
 678
 679         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
 680         if (power_well->count > 0)
 681                 bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 682
 683         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC);
 684         if (power_well->count > 0)
 685                 bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 686
 687         if (IS_GEMINILAKE(dev_priv)) {
 688                 power_well = lookup_power_well(dev_priv, GLK_DPIO_CMN_C);
 689                 if (power_well->count > 0)
 690                         bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 691         }
 692 }
 693
 694 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
 695                                            struct i915_power_well *power_well)
 696 {
 697         return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
 698 }
 699
 700 static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
 701 {
 702         u32 tmp = I915_READ(DBUF_CTL);
 703
 704         WARN((tmp & (DBUF_POWER_STATE | DBUF_POWER_REQUEST)) !=
 705              (DBUF_POWER_STATE | DBUF_POWER_REQUEST),
 706              "Unexpected DBuf power power state (0x%08x)\n", tmp);
 707 }
 708
 709 static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
 710                                           struct i915_power_well *power_well)
 711 {
 712         struct intel_cdclk_state cdclk_state = {};
 713
 714         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 715
 716         dev_priv->display.get_cdclk(dev_priv, &cdclk_state);
 717         WARN_ON(!intel_cdclk_state_compare(&dev_priv->cdclk.hw, &cdclk_state));
 718
 719         gen9_assert_dbuf_enabled(dev_priv);
 720
 721         if (IS_GEN9_LP(dev_priv))
 722                 bxt_verify_ddi_phy_power_wells(dev_priv);
 723 }
 724
 725 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
 726                                            struct i915_power_well *power_well)
 727 {
 728         if (!dev_priv->csr.dmc_payload)
 729                 return;
 730
 731         if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
 732                 skl_enable_dc6(dev_priv);
 733         else if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
 734                 gen9_enable_dc5(dev_priv);
 735 }
 736
 737 static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
 738                                          struct i915_power_well *power_well)
 739 {
 740 }
 741
 742 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
 743                                            struct i915_power_well *power_well)
 744 {
 745 }
 746
 747 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 748                                              struct i915_power_well *power_well)
 749 {
 750         return true;
 751 }
 752
 753 static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
 754                                          struct i915_power_well *power_well)
 755 {
 756         if ((I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE) == 0)
 757                 i830_enable_pipe(dev_priv, PIPE_A);
 758         if ((I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE) == 0)
 759                 i830_enable_pipe(dev_priv, PIPE_B);
 760 }
 761
 762 static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
 763                                           struct i915_power_well *power_well)
 764 {
 765         i830_disable_pipe(dev_priv, PIPE_B);
 766         i830_disable_pipe(dev_priv, PIPE_A);
 767 }
 768
 769 static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
 770                                           struct i915_power_well *power_well)
 771 {
 772         return I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE &&
 773                 I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
 774 }
 775
 776 static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
 777                                           struct i915_power_well *power_well)
 778 {
 779         if (power_well->count > 0)
 780                 i830_pipes_power_well_enable(dev_priv, power_well);
 781         else
 782                 i830_pipes_power_well_disable(dev_priv, power_well);
 783 }
 784
 785 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 786                                struct i915_power_well *power_well, bool enable)
 787 {
 788         enum i915_power_well_id power_well_id = power_well->id;
 789         u32 mask;
 790         u32 state;
 791         u32 ctrl;
 792
 793         mask = PUNIT_PWRGT_MASK(power_well_id);
 794         state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
 795                          PUNIT_PWRGT_PWR_GATE(power_well_id);
 796
 797         mutex_lock(&dev_priv->rps.hw_lock);
 798
 799 #define COND \
 800         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
 801
 802         if (COND)
 803                 goto out;
 804
 805         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
 806         ctrl &= ~mask;
 807         ctrl |= state;
 808         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
 809
 810         if (wait_for(COND, 100))
 811                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
 812                           state,
 813                           vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
 814
 815 #undef COND
 816
 817 out:
 818         mutex_unlock(&dev_priv->rps.hw_lock);
 819 }
 820
 821 static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
 822                                   struct i915_power_well *power_well)
 823 {
 824         vlv_set_power_well(dev_priv, power_well, true);
 825 }
 826
 827 static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
 828                                    struct i915_power_well *power_well)
 829 {
 830         vlv_set_power_well(dev_priv, power_well, false);
 831 }
 832
 833 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
 834                                    struct i915_power_well *power_well)
 835 {
 836         enum i915_power_well_id power_well_id = power_well->id;
 837         bool enabled = false;
 838         u32 mask;
 839         u32 state;
 840         u32 ctrl;
 841
 842         mask = PUNIT_PWRGT_MASK(power_well_id);
 843         ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
 844
 845         mutex_lock(&dev_priv->rps.hw_lock);
 846
 847         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
 848         /*
 849          * We only ever set the power-on and power-gate states, anything
 850          * else is unexpected.
 851          */
 852         WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
 853                 state != PUNIT_PWRGT_PWR_GATE(power_well_id));
 854         if (state == ctrl)
 855                 enabled = true;
 856
 857         /*
 858          * A transient state at this point would mean some unexpected party
 859          * is poking at the power controls too.
 860          */
 861         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
 862         WARN_ON(ctrl != state);
 863
 864         mutex_unlock(&dev_priv->rps.hw_lock);
 865
 866         return enabled;
 867 }
 868
 869 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
 870 {
 871         u32 val;
 872
 873         /*
 874          * On driver load, a pipe may be active and driving a DSI display.
 875          * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
 876          * (and never recovering) in this case. intel_dsi_post_disable() will
 877          * clear it when we turn off the display.
 878          */
 879         val = I915_READ(DSPCLK_GATE_D);
 880         val &= DPOUNIT_CLOCK_GATE_DISABLE;
 881         val |= VRHUNIT_CLOCK_GATE_DISABLE;
 882         I915_WRITE(DSPCLK_GATE_D, val);
 883
 884         /*
 885          * Disable trickle feed and enable pnd deadline calculation
 886          */
 887         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
 888         I915_WRITE(CBR1_VLV, 0);
 889
 890         WARN_ON(dev_priv->rawclk_freq == 0);
 891
 892         I915_WRITE(RAWCLK_FREQ_VLV,
 893                    DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 1000));
 894 }
 895
 896 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 897 {
 898         struct intel_encoder *encoder;
 899         enum pipe pipe;
 900
 901         /*
 902          * Enable the CRI clock source so we can get at the
 903          * display and the reference clock for VGA
 904          * hotplug / manual detection. Supposedly DSI also
 905          * needs the ref clock up and running.
 906          *
 907          * CHV DPLL B/C have some issues if VGA mode is enabled.
 908          */
 909         for_each_pipe(dev_priv, pipe) {
 910                 u32 val = I915_READ(DPLL(pipe));
 911
 912                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
 913                 if (pipe != PIPE_A)
 914                         val |= DPLL_INTEGRATED_CRI_CLK_VLV;
 915
 916                 I915_WRITE(DPLL(pipe), val);
 917         }
 918
 919         vlv_init_display_clock_gating(dev_priv);
 920
 921         spin_lock_irq(&dev_priv->irq_lock);
 922         valleyview_enable_display_irqs(dev_priv);
 923         spin_unlock_irq(&dev_priv->irq_lock);
 924
 925         /*
 926          * During driver initialization/resume we can avoid restoring the
 927          * part of the HW/SW state that will be inited anyway explicitly.
 928          */
 929         if (dev_priv->power_domains.initializing)
 930                 return;
 931
 932         intel_hpd_init(dev_priv);
 933
 934         /* Re-enable the ADPA, if we have one */
 935         for_each_intel_encoder(&dev_priv->drm, encoder) {
 936                 if (encoder->type == INTEL_OUTPUT_ANALOG)
 937                         intel_crt_reset(&encoder->base);
 938         }
 939
 940         i915_redisable_vga_power_on(dev_priv);
 941
 942         intel_pps_unlock_regs_wa(dev_priv);
 943 }
 944
 945 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
 946 {
 947         spin_lock_irq(&dev_priv->irq_lock);
 948         valleyview_disable_display_irqs(dev_priv);
 949         spin_unlock_irq(&dev_priv->irq_lock);
 950
 951         /* make sure we're done processing display irqs */
 952         synchronize_irq(dev_priv->drm.irq);
 953
 954         intel_power_sequencer_reset(dev_priv);
 955
 956         /* Prevent us from re-enabling polling on accident in late suspend */
 957         if (!dev_priv->drm.dev->power.is_suspended)
 958                 intel_hpd_poll_init(dev_priv);
 959 }
 960
 961 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
 962                                           struct i915_power_well *power_well)
 963 {
 964         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 965
 966         vlv_set_power_well(dev_priv, power_well, true);
 967
 968         vlv_display_power_well_init(dev_priv);
 969 }
 970
 971 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
 972                                            struct i915_power_well *power_well)
 973 {
 974         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 975
 976         vlv_display_power_well_deinit(dev_priv);
 977
 978         vlv_set_power_well(dev_priv, power_well, false);
 979 }
 980
 981 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 982                                            struct i915_power_well *power_well)
 983 {
 984         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
 985
 986         /* since ref/cri clock was enabled */
 987         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
 988
 989         vlv_set_power_well(dev_priv, power_well, true);
 990
 991         /*
 992          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
 993          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
 994          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
 995          *   b. The other bits such as sfr settings / modesel may all
 996          *      be set to 0.
 997          *
 998          * This should only be done on init and resume from S3 with
 999          * both PLLs disabled, or we risk losing DPIO and PLL
1000          * synchronization.
1001          */
1002         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
1003 }
1004
1005 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1006                                             struct i915_power_well *power_well)
1007 {
1008         enum pipe pipe;
1009
1010         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
1011
1012         for_each_pipe(dev_priv, pipe)
1013                 assert_pll_disabled(dev_priv, pipe);
1014
1015         /* Assert common reset */
1016         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
1017
1018         vlv_set_power_well(dev_priv, power_well, false);
1019 }
1020
1021 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
1022
1023 static struct i915_power_well *
1024 lookup_power_well(struct drm_i915_private *dev_priv,
1025                   enum i915_power_well_id power_well_id)
1026 {
1027         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1028         int i;
1029
1030         for (i = 0; i < power_domains->power_well_count; i++) {
1031                 struct i915_power_well *power_well;
1032
1033                 power_well = &power_domains->power_wells[i];
1034                 if (power_well->id == power_well_id)
1035                         return power_well;
1036         }
1037
1038         return NULL;
1039 }
1040
1041 #define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1042
1043 static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1044 {
1045         struct i915_power_well *cmn_bc =
1046                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
1047         struct i915_power_well *cmn_d =
1048                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
1049         u32 phy_control = dev_priv->chv_phy_control;
1050         u32 phy_status = 0;
1051         u32 phy_status_mask = 0xffffffff;
1052
1053         /*
1054          * The BIOS can leave the PHY is some weird state
1055          * where it doesn't fully power down some parts.
1056          * Disable the asserts until the PHY has been fully
1057          * reset (ie. the power well has been disabled at
1058          * least once).
1059          */
1060         if (!dev_priv->chv_phy_assert[DPIO_PHY0])
1061                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1062                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1063                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1064                                      PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1065                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1066                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1067
1068         if (!dev_priv->chv_phy_assert[DPIO_PHY1])
1069                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1070                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1071                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1072
1073         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
1074                 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1075
1076                 /* this assumes override is only used to enable lanes */
1077                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1078                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1079
1080                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1081                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1082
1083                 /* CL1 is on whenever anything is on in either channel */
1084                 if (BITS_SET(phy_control,
1085                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1086                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1087                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1088
1089                 /*
1090                  * The DPLLB check accounts for the pipe B + port A usage
1091                  * with CL2 powered up but all the lanes in the second channel
1092                  * powered down.
1093                  */
1094                 if (BITS_SET(phy_control,
1095                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1096                     (I915_READ(DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1097                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1098
1099                 if (BITS_SET(phy_control,
1100                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1101                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1102                 if (BITS_SET(phy_control,
1103                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1104                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1105
1106                 if (BITS_SET(phy_control,
1107                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1108                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1109                 if (BITS_SET(phy_control,
1110                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1111                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1112         }
1113
1114         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
1115                 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1116
1117                 /* this assumes override is only used to enable lanes */
1118                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1119                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1120
1121                 if (BITS_SET(phy_control,
1122                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1123                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1124
1125                 if (BITS_SET(phy_control,
1126                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1127                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1128                 if (BITS_SET(phy_control,
1129                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1130                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1131         }
1132
1133         phy_status &= phy_status_mask;
1134
1135         /*
1136          * The PHY may be busy with some initial calibration and whatnot,
1137          * so the power state can take a while to actually change.
1138          */
1139         if (intel_wait_for_register(dev_priv,
1140                                     DISPLAY_PHY_STATUS,
1141                                     phy_status_mask,
1142                                     phy_status,
1143                                     10))
1144                 DRM_ERROR("Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1145                           I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask,
1146                            phy_status, dev_priv->chv_phy_control);
1147 }
1148
1149 #undef BITS_SET
1150
1151 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1152                                            struct i915_power_well *power_well)
1153 {
1154         enum dpio_phy phy;
1155         enum pipe pipe;
1156         uint32_t tmp;
1157
1158         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1159                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1160
1161         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1162                 pipe = PIPE_A;
1163                 phy = DPIO_PHY0;
1164         } else {
1165                 pipe = PIPE_C;
1166                 phy = DPIO_PHY1;
1167         }
1168
1169         /* since ref/cri clock was enabled */
1170         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1171         vlv_set_power_well(dev_priv, power_well, true);
1172
1173         /* Poll for phypwrgood signal */
1174         if (intel_wait_for_register(dev_priv,
1175                                     DISPLAY_PHY_STATUS,
1176                                     PHY_POWERGOOD(phy),
1177                                     PHY_POWERGOOD(phy),
1178                                     1))
1179                 DRM_ERROR("Display PHY %d is not power up\n", phy);
1180
1181         mutex_lock(&dev_priv->sb_lock);
1182
1183         /* Enable dynamic power down */
1184         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
1185         tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1186                 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1187         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
1188
1189         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1190                 tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
1191                 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1192                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
1193         } else {
1194                 /*
1195                  * Force the non-existing CL2 off. BXT does this
1196                  * too, so maybe it saves some power even though
1197                  * CL2 doesn't exist?
1198                  */
1199                 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
1200                 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1201                 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
1202         }
1203
1204         mutex_unlock(&dev_priv->sb_lock);
1205
1206         dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1207         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1208
1209         DRM_DEBUG_KMS("Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1210                       phy, dev_priv->chv_phy_control);
1211
1212         assert_chv_phy_status(dev_priv);
1213 }
1214
1215 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1216                                             struct i915_power_well *power_well)
1217 {
1218         enum dpio_phy phy;
1219
1220         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1221                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1222
1223         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1224                 phy = DPIO_PHY0;
1225                 assert_pll_disabled(dev_priv, PIPE_A);
1226                 assert_pll_disabled(dev_priv, PIPE_B);
1227         } else {
1228                 phy = DPIO_PHY1;
1229                 assert_pll_disabled(dev_priv, PIPE_C);
1230         }
1231
1232         dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1233         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1234
1235         vlv_set_power_well(dev_priv, power_well, false);
1236
1237         DRM_DEBUG_KMS("Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1238                       phy, dev_priv->chv_phy_control);
1239
1240         /* PHY is fully reset now, so we can enable the PHY state asserts */
1241         dev_priv->chv_phy_assert[phy] = true;
1242
1243         assert_chv_phy_status(dev_priv);
1244 }
1245
1246 static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1247                                      enum dpio_channel ch, bool override, unsigned int mask)
1248 {
1249         enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
1250         u32 reg, val, expected, actual;
1251
1252         /*
1253          * The BIOS can leave the PHY is some weird state
1254          * where it doesn't fully power down some parts.
1255          * Disable the asserts until the PHY has been fully
1256          * reset (ie. the power well has been disabled at
1257          * least once).
1258          */
1259         if (!dev_priv->chv_phy_assert[phy])
1260                 return;
1261
1262         if (ch == DPIO_CH0)
1263                 reg = _CHV_CMN_DW0_CH0;
1264         else
1265                 reg = _CHV_CMN_DW6_CH1;
1266
1267         mutex_lock(&dev_priv->sb_lock);
1268         val = vlv_dpio_read(dev_priv, pipe, reg);
1269         mutex_unlock(&dev_priv->sb_lock);
1270
1271         /*
1272          * This assumes !override is only used when the port is disabled.
1273          * All lanes should power down even without the override when
1274          * the port is disabled.
1275          */
1276         if (!override || mask == 0xf) {
1277                 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1278                 /*
1279                  * If CH1 common lane is not active anymore
1280                  * (eg. for pipe B DPLL) the entire channel will
1281                  * shut down, which causes the common lane registers
1282                  * to read as 0. That means we can't actually check
1283                  * the lane power down status bits, but as the entire
1284                  * register reads as 0 it's a good indication that the
1285                  * channel is indeed entirely powered down.
1286                  */
1287                 if (ch == DPIO_CH1 && val == 0)
1288                         expected = 0;
1289         } else if (mask != 0x0) {
1290                 expected = DPIO_ANYDL_POWERDOWN;
1291         } else {
1292                 expected = 0;
1293         }
1294
1295         if (ch == DPIO_CH0)
1296                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1297         else
1298                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1299         actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1300
1301         WARN(actual != expected,
1302              "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1303              !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN),
1304              !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN),
1305              reg, val);
1306 }
1307
1308 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1309                           enum dpio_channel ch, bool override)
1310 {
1311         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1312         bool was_override;
1313
1314         mutex_lock(&power_domains->lock);
1315
1316         was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1317
1318         if (override == was_override)
1319                 goto out;
1320
1321         if (override)
1322                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1323         else
1324                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1325
1326         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1327
1328         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1329                       phy, ch, dev_priv->chv_phy_control);
1330
1331         assert_chv_phy_status(dev_priv);
1332
1333 out:
1334         mutex_unlock(&power_domains->lock);
1335
1336         return was_override;
1337 }
1338
1339 void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1340                              bool override, unsigned int mask)
1341 {
1342         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1343         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1344         enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(&encoder->base));
1345         enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
1346
1347         mutex_lock(&power_domains->lock);
1348
1349         dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1350         dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1351
1352         if (override)
1353                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1354         else
1355                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1356
1357         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1358
1359         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1360                       phy, ch, mask, dev_priv->chv_phy_control);
1361
1362         assert_chv_phy_status(dev_priv);
1363
1364         assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1365
1366         mutex_unlock(&power_domains->lock);
1367 }
1368
1369 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1370                                         struct i915_power_well *power_well)
1371 {
1372         enum pipe pipe = PIPE_A;
1373         bool enabled;
1374         u32 state, ctrl;
1375
1376         mutex_lock(&dev_priv->rps.hw_lock);
1377
1378         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
1379         /*
1380          * We only ever set the power-on and power-gate states, anything
1381          * else is unexpected.
1382          */
1383         WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
1384         enabled = state == DP_SSS_PWR_ON(pipe);
1385
1386         /*
1387          * A transient state at this point would mean some unexpected party
1388          * is poking at the power controls too.
1389          */
1390         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
1391         WARN_ON(ctrl << 16 != state);
1392
1393         mutex_unlock(&dev_priv->rps.hw_lock);
1394
1395         return enabled;
1396 }
1397
1398 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1399                                     struct i915_power_well *power_well,
1400                                     bool enable)
1401 {
1402         enum pipe pipe = PIPE_A;
1403         u32 state;
1404         u32 ctrl;
1405
1406         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1407
1408         mutex_lock(&dev_priv->rps.hw_lock);
1409
1410 #define COND \
1411         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
1412
1413         if (COND)
1414                 goto out;
1415
1416         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
1417         ctrl &= ~DP_SSC_MASK(pipe);
1418         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1419         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
1420
1421         if (wait_for(COND, 100))
1422                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
1423                           state,
1424                           vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
1425
1426 #undef COND
1427
1428 out:
1429         mutex_unlock(&dev_priv->rps.hw_lock);
1430 }
1431
1432 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1433                                        struct i915_power_well *power_well)
1434 {
1435         WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
1436
1437         chv_set_pipe_power_well(dev_priv, power_well, true);
1438
1439         vlv_display_power_well_init(dev_priv);
1440 }
1441
1442 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1443                                         struct i915_power_well *power_well)
1444 {
1445         WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
1446
1447         vlv_display_power_well_deinit(dev_priv);
1448
1449         chv_set_pipe_power_well(dev_priv, power_well, false);
1450 }
1451
1452 static void
1453 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
1454                                  enum intel_display_power_domain domain)
1455 {
1456         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1457         struct i915_power_well *power_well;
1458
1459         for_each_power_domain_well(dev_priv, power_well, BIT_ULL(domain))
1460                 intel_power_well_get(dev_priv, power_well);
1461
1462         power_domains->domain_use_count[domain]++;
1463 }
1464
1465 /**
1466  * intel_display_power_get - grab a power domain reference
1467  * @dev_priv: i915 device instance
1468  * @domain: power domain to reference
1469  *
1470  * This function grabs a power domain reference for @domain and ensures that the
1471  * power domain and all its parents are powered up. Therefore users should only
1472  * grab a reference to the innermost power domain they need.
1473  *
1474  * Any power domain reference obtained by this function must have a symmetric
1475  * call to intel_display_power_put() to release the reference again.
1476  */
1477 void intel_display_power_get(struct drm_i915_private *dev_priv,
1478                              enum intel_display_power_domain domain)
1479 {
1480         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1481
1482         intel_runtime_pm_get(dev_priv);
1483
1484         mutex_lock(&power_domains->lock);
1485
1486         __intel_display_power_get_domain(dev_priv, domain);
1487
1488         mutex_unlock(&power_domains->lock);
1489 }
1490
1491 /**
1492  * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
1493  * @dev_priv: i915 device instance
1494  * @domain: power domain to reference
1495  *
1496  * This function grabs a power domain reference for @domain and ensures that the
1497  * power domain and all its parents are powered up. Therefore users should only
1498  * grab a reference to the innermost power domain they need.
1499  *
1500  * Any power domain reference obtained by this function must have a symmetric
1501  * call to intel_display_power_put() to release the reference again.
1502  */
1503 bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
1504                                         enum intel_display_power_domain domain)
1505 {
1506         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1507         bool is_enabled;
1508
1509         if (!intel_runtime_pm_get_if_in_use(dev_priv))
1510                 return false;
1511
1512         mutex_lock(&power_domains->lock);
1513
1514         if (__intel_display_power_is_enabled(dev_priv, domain)) {
1515                 __intel_display_power_get_domain(dev_priv, domain);
1516                 is_enabled = true;
1517         } else {
1518                 is_enabled = false;
1519         }
1520
1521         mutex_unlock(&power_domains->lock);
1522
1523         if (!is_enabled)
1524                 intel_runtime_pm_put(dev_priv);
1525
1526         return is_enabled;
1527 }
1528
1529 /**
1530  * intel_display_power_put - release a power domain reference
1531  * @dev_priv: i915 device instance
1532  * @domain: power domain to reference
1533  *
1534  * This function drops the power domain reference obtained by
1535  * intel_display_power_get() and might power down the corresponding hardware
1536  * block right away if this is the last reference.
1537  */
1538 void intel_display_power_put(struct drm_i915_private *dev_priv,
1539                              enum intel_display_power_domain domain)
1540 {
1541         struct i915_power_domains *power_domains;
1542         struct i915_power_well *power_well;
1543
1544         power_domains = &dev_priv->power_domains;
1545
1546         mutex_lock(&power_domains->lock);
1547
1548         WARN(!power_domains->domain_use_count[domain],
1549              "Use count on domain %s is already zero\n",
1550              intel_display_power_domain_str(domain));
1551         power_domains->domain_use_count[domain]--;
1552
1553         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain))
1554                 intel_power_well_put(dev_priv, power_well);
1555
1556         mutex_unlock(&power_domains->lock);
1557
1558         intel_runtime_pm_put(dev_priv);
1559 }
1560
1561 #define I830_PIPES_POWER_DOMAINS (              \
1562         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1563         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1564         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1565         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1566         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1567         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1568         BIT_ULL(POWER_DOMAIN_INIT))
1569
1570 #define VLV_DISPLAY_POWER_DOMAINS (             \
1571         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1572         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1573         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1574         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1575         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1576         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1577         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1578         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1579         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1580         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1581         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1582         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1583         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1584         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1585         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1586         BIT_ULL(POWER_DOMAIN_INIT))
1587
1588 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
1589         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1590         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1591         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1592         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1593         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1594         BIT_ULL(POWER_DOMAIN_INIT))
1595
1596 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
1597         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1598         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1599         BIT_ULL(POWER_DOMAIN_INIT))
1600
1601 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
1602         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1603         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1604         BIT_ULL(POWER_DOMAIN_INIT))
1605
1606 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
1607         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1608         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1609         BIT_ULL(POWER_DOMAIN_INIT))
1610
1611 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
1612         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1613         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1614         BIT_ULL(POWER_DOMAIN_INIT))
1615
1616 #define CHV_DISPLAY_POWER_DOMAINS (             \
1617         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1618         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1619         BIT_ULL(POWER_DOMAIN_PIPE_C) |          \
1620         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1621         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1622         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
1623         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1624         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1625         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |    \
1626         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1627         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1628         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1629         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1630         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1631         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1632         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1633         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1634         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1635         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1636         BIT_ULL(POWER_DOMAIN_INIT))
1637
1638 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
1639         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1640         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1641         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1642         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1643         BIT_ULL(POWER_DOMAIN_INIT))
1644
1645 #define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
1646         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1647         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1648         BIT_ULL(POWER_DOMAIN_INIT))
1649
1650 #define HSW_DISPLAY_POWER_DOMAINS (                     \
1651         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1652         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1653         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |             \
1654         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1655         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1656         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1657         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1658         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1659         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1660         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1661         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1662         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1663         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1664         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1665         BIT_ULL(POWER_DOMAIN_INIT))
1666
1667 #define BDW_DISPLAY_POWER_DOMAINS (                     \
1668         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1669         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1670         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1671         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1672         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1673         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1674         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1675         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1676         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1677         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1678         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1679         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1680         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1681         BIT_ULL(POWER_DOMAIN_INIT))
1682
1683 #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1684         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1685         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1686         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1687         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1688         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1689         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1690         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1691         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1692         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1693         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1694         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |                \
1695         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1696         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1697         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1698         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1699         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1700         BIT_ULL(POWER_DOMAIN_INIT))
1701 #define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS (          \
1702         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1703         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |           \
1704         BIT_ULL(POWER_DOMAIN_INIT))
1705 #define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1706         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1707         BIT_ULL(POWER_DOMAIN_INIT))
1708 #define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1709         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1710         BIT_ULL(POWER_DOMAIN_INIT))
1711 #define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS (            \
1712         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1713         BIT_ULL(POWER_DOMAIN_INIT))
1714 #define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1715         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1716         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1717         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1718         BIT_ULL(POWER_DOMAIN_INIT))
1719
1720 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1721         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1722         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1723         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1724         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1725         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1726         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1727         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1728         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1729         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1730         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1731         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1732         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1733         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1734         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
1735         BIT_ULL(POWER_DOMAIN_INIT))
1736 #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1737         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1738         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1739         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1740         BIT_ULL(POWER_DOMAIN_INIT))
1741 #define BXT_DPIO_CMN_A_POWER_DOMAINS (                  \
1742         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1743         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1744         BIT_ULL(POWER_DOMAIN_INIT))
1745 #define BXT_DPIO_CMN_BC_POWER_DOMAINS (                 \
1746         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1747         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1748         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1749         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1750         BIT_ULL(POWER_DOMAIN_INIT))
1751
1752 #define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1753         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1754         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1755         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1756         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1757         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1758         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1759         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1760         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1761         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1762         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1763         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1764         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1765         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1766         BIT_ULL(POWER_DOMAIN_INIT))
1767 #define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS (            \
1768         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
1769 #define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1770         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
1771 #define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1772         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
1773 #define GLK_DPIO_CMN_A_POWER_DOMAINS (                  \
1774         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1775         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1776         BIT_ULL(POWER_DOMAIN_INIT))
1777 #define GLK_DPIO_CMN_B_POWER_DOMAINS (                  \
1778         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1779         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1780         BIT_ULL(POWER_DOMAIN_INIT))
1781 #define GLK_DPIO_CMN_C_POWER_DOMAINS (                  \
1782         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1783         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1784         BIT_ULL(POWER_DOMAIN_INIT))
1785 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (               \
1786         BIT_ULL(POWER_DOMAIN_AUX_A) |           \
1787         BIT_ULL(POWER_DOMAIN_INIT))
1788 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (               \
1789         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1790         BIT_ULL(POWER_DOMAIN_INIT))
1791 #define GLK_DISPLAY_AUX_C_POWER_DOMAINS (               \
1792         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1793         BIT_ULL(POWER_DOMAIN_INIT))
1794 #define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1795         GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1796         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1797         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1798         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
1799         BIT_ULL(POWER_DOMAIN_INIT))
1800
1801 #define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1802         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1803         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1804         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1805         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1806         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1807         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1808         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1809         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1810         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1811         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1812         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1813         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1814         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1815         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1816         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1817         BIT_ULL(POWER_DOMAIN_INIT))
1818 #define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (            \
1819         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1820         BIT_ULL(POWER_DOMAIN_INIT))
1821 #define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (            \
1822         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1823         BIT_ULL(POWER_DOMAIN_INIT))
1824 #define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (            \
1825         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1826         BIT_ULL(POWER_DOMAIN_INIT))
1827 #define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (            \
1828         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1829         BIT_ULL(POWER_DOMAIN_INIT))
1830 #define CNL_DISPLAY_AUX_A_POWER_DOMAINS (               \
1831         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1832         BIT_ULL(POWER_DOMAIN_INIT))
1833 #define CNL_DISPLAY_AUX_B_POWER_DOMAINS (               \
1834         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1835         BIT_ULL(POWER_DOMAIN_INIT))
1836 #define CNL_DISPLAY_AUX_C_POWER_DOMAINS (               \
1837         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1838         BIT_ULL(POWER_DOMAIN_INIT))
1839 #define CNL_DISPLAY_AUX_D_POWER_DOMAINS (               \
1840         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1841         BIT_ULL(POWER_DOMAIN_INIT))
1842 #define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1843         CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1844         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1845         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1846         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
1847         BIT_ULL(POWER_DOMAIN_INIT))
1848
1849 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
1850         .sync_hw = i9xx_power_well_sync_hw_noop,
1851         .enable = i9xx_always_on_power_well_noop,
1852         .disable = i9xx_always_on_power_well_noop,
1853         .is_enabled = i9xx_always_on_power_well_enabled,
1854 };
1855
1856 static const struct i915_power_well_ops chv_pipe_power_well_ops = {
1857         .sync_hw = i9xx_power_well_sync_hw_noop,
1858         .enable = chv_pipe_power_well_enable,
1859         .disable = chv_pipe_power_well_disable,
1860         .is_enabled = chv_pipe_power_well_enabled,
1861 };
1862
1863 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
1864         .sync_hw = i9xx_power_well_sync_hw_noop,
1865         .enable = chv_dpio_cmn_power_well_enable,
1866         .disable = chv_dpio_cmn_power_well_disable,
1867         .is_enabled = vlv_power_well_enabled,
1868 };
1869
1870 static struct i915_power_well i9xx_always_on_power_well[] = {
1871         {
1872                 .name = "always-on",
1873                 .always_on = 1,
1874                 .domains = POWER_DOMAIN_MASK,
1875                 .ops = &i9xx_always_on_power_well_ops,
1876                 .id = I915_DISP_PW_ALWAYS_ON,
1877         },
1878 };
1879
1880 static const struct i915_power_well_ops i830_pipes_power_well_ops = {
1881         .sync_hw = i830_pipes_power_well_sync_hw,
1882         .enable = i830_pipes_power_well_enable,
1883         .disable = i830_pipes_power_well_disable,
1884         .is_enabled = i830_pipes_power_well_enabled,
1885 };
1886
1887 static struct i915_power_well i830_power_wells[] = {
1888         {
1889                 .name = "always-on",
1890                 .always_on = 1,
1891                 .domains = POWER_DOMAIN_MASK,
1892                 .ops = &i9xx_always_on_power_well_ops,
1893                 .id = I915_DISP_PW_ALWAYS_ON,
1894         },
1895         {
1896                 .name = "pipes",
1897                 .domains = I830_PIPES_POWER_DOMAINS,
1898                 .ops = &i830_pipes_power_well_ops,
1899                 .id = I830_DISP_PW_PIPES,
1900         },
1901 };
1902
1903 static const struct i915_power_well_ops hsw_power_well_ops = {
1904         .sync_hw = hsw_power_well_sync_hw,
1905         .enable = hsw_power_well_enable,
1906         .disable = hsw_power_well_disable,
1907         .is_enabled = hsw_power_well_enabled,
1908 };
1909
1910 static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
1911         .sync_hw = i9xx_power_well_sync_hw_noop,
1912         .enable = gen9_dc_off_power_well_enable,
1913         .disable = gen9_dc_off_power_well_disable,
1914         .is_enabled = gen9_dc_off_power_well_enabled,
1915 };
1916
1917 static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
1918         .sync_hw = i9xx_power_well_sync_hw_noop,
1919         .enable = bxt_dpio_cmn_power_well_enable,
1920         .disable = bxt_dpio_cmn_power_well_disable,
1921         .is_enabled = bxt_dpio_cmn_power_well_enabled,
1922 };
1923
1924 static struct i915_power_well hsw_power_wells[] = {
1925         {
1926                 .name = "always-on",
1927                 .always_on = 1,
1928                 .domains = POWER_DOMAIN_MASK,
1929                 .ops = &i9xx_always_on_power_well_ops,
1930                 .id = I915_DISP_PW_ALWAYS_ON,
1931         },
1932         {
1933                 .name = "display",
1934                 .domains = HSW_DISPLAY_POWER_DOMAINS,
1935                 .ops = &hsw_power_well_ops,
1936                 .id = HSW_DISP_PW_GLOBAL,
1937                 {
1938                         .hsw.has_vga = true,
1939                 },
1940         },
1941 };
1942
1943 static struct i915_power_well bdw_power_wells[] = {
1944         {
1945                 .name = "always-on",
1946                 .always_on = 1,
1947                 .domains = POWER_DOMAIN_MASK,
1948                 .ops = &i9xx_always_on_power_well_ops,
1949                 .id = I915_DISP_PW_ALWAYS_ON,
1950         },
1951         {
1952                 .name = "display",
1953                 .domains = BDW_DISPLAY_POWER_DOMAINS,
1954                 .ops = &hsw_power_well_ops,
1955                 .id = HSW_DISP_PW_GLOBAL,
1956                 {
1957                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
1958                         .hsw.has_vga = true,
1959                 },
1960         },
1961 };
1962
1963 static const struct i915_power_well_ops vlv_display_power_well_ops = {
1964         .sync_hw = i9xx_power_well_sync_hw_noop,
1965         .enable = vlv_display_power_well_enable,
1966         .disable = vlv_display_power_well_disable,
1967         .is_enabled = vlv_power_well_enabled,
1968 };
1969
1970 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
1971         .sync_hw = i9xx_power_well_sync_hw_noop,
1972         .enable = vlv_dpio_cmn_power_well_enable,
1973         .disable = vlv_dpio_cmn_power_well_disable,
1974         .is_enabled = vlv_power_well_enabled,
1975 };
1976
1977 static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
1978         .sync_hw = i9xx_power_well_sync_hw_noop,
1979         .enable = vlv_power_well_enable,
1980         .disable = vlv_power_well_disable,
1981         .is_enabled = vlv_power_well_enabled,
1982 };
1983
1984 static struct i915_power_well vlv_power_wells[] = {
1985         {
1986                 .name = "always-on",
1987                 .always_on = 1,
1988                 .domains = POWER_DOMAIN_MASK,
1989                 .ops = &i9xx_always_on_power_well_ops,
1990                 .id = I915_DISP_PW_ALWAYS_ON,
1991         },
1992         {
1993                 .name = "display",
1994                 .domains = VLV_DISPLAY_POWER_DOMAINS,
1995                 .id = PUNIT_POWER_WELL_DISP2D,
1996                 .ops = &vlv_display_power_well_ops,
1997         },
1998         {
1999                 .name = "dpio-tx-b-01",
2000                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2001                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2002                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2003                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2004                 .ops = &vlv_dpio_power_well_ops,
2005                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
2006         },
2007         {
2008                 .name = "dpio-tx-b-23",
2009                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2010                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2011                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2012                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2013                 .ops = &vlv_dpio_power_well_ops,
2014                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
2015         },
2016         {
2017                 .name = "dpio-tx-c-01",
2018                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2019                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2020                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2021                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2022                 .ops = &vlv_dpio_power_well_ops,
2023                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
2024         },
2025         {
2026                 .name = "dpio-tx-c-23",
2027                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2028                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2029                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2030                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2031                 .ops = &vlv_dpio_power_well_ops,
2032                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
2033         },
2034         {
2035                 .name = "dpio-common",
2036                 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
2037                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2038                 .ops = &vlv_dpio_cmn_power_well_ops,
2039         },
2040 };
2041
2042 static struct i915_power_well chv_power_wells[] = {
2043         {
2044                 .name = "always-on",
2045                 .always_on = 1,
2046                 .domains = POWER_DOMAIN_MASK,
2047                 .ops = &i9xx_always_on_power_well_ops,
2048                 .id = I915_DISP_PW_ALWAYS_ON,
2049         },
2050         {
2051                 .name = "display",
2052                 /*
2053                  * Pipe A power well is the new disp2d well. Pipe B and C
2054                  * power wells don't actually exist. Pipe A power well is
2055                  * required for any pipe to work.
2056                  */
2057                 .domains = CHV_DISPLAY_POWER_DOMAINS,
2058                 .id = CHV_DISP_PW_PIPE_A,
2059                 .ops = &chv_pipe_power_well_ops,
2060         },
2061         {
2062                 .name = "dpio-common-bc",
2063                 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
2064                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2065                 .ops = &chv_dpio_cmn_power_well_ops,
2066         },
2067         {
2068                 .name = "dpio-common-d",
2069                 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
2070                 .id = PUNIT_POWER_WELL_DPIO_CMN_D,
2071                 .ops = &chv_dpio_cmn_power_well_ops,
2072         },
2073 };
2074
2075 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
2076                                          enum i915_power_well_id power_well_id)
2077 {
2078         struct i915_power_well *power_well;
2079         bool ret;
2080
2081         power_well = lookup_power_well(dev_priv, power_well_id);
2082         ret = power_well->ops->is_enabled(dev_priv, power_well);
2083
2084         return ret;
2085 }
2086
2087 static struct i915_power_well skl_power_wells[] = {
2088         {
2089                 .name = "always-on",
2090                 .always_on = 1,
2091                 .domains = POWER_DOMAIN_MASK,
2092                 .ops = &i9xx_always_on_power_well_ops,
2093                 .id = I915_DISP_PW_ALWAYS_ON,
2094         },
2095         {
2096                 .name = "power well 1",
2097                 /* Handled by the DMC firmware */
2098                 .domains = 0,
2099                 .ops = &hsw_power_well_ops,
2100                 .id = SKL_DISP_PW_1,
2101                 {
2102                         .hsw.has_fuses = true,
2103                 },
2104         },
2105         {
2106                 .name = "MISC IO power well",
2107                 /* Handled by the DMC firmware */
2108                 .domains = 0,
2109                 .ops = &hsw_power_well_ops,
2110                 .id = SKL_DISP_PW_MISC_IO,
2111         },
2112         {
2113                 .name = "DC off",
2114                 .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
2115                 .ops = &gen9_dc_off_power_well_ops,
2116                 .id = SKL_DISP_PW_DC_OFF,
2117         },
2118         {
2119                 .name = "power well 2",
2120                 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2121                 .ops = &hsw_power_well_ops,
2122                 .id = SKL_DISP_PW_2,
2123                 {
2124                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2125                         .hsw.has_vga = true,
2126                         .hsw.has_fuses = true,
2127                 },
2128         },
2129         {
2130                 .name = "DDI A/E IO power well",
2131                 .domains = SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS,
2132                 .ops = &hsw_power_well_ops,
2133                 .id = SKL_DISP_PW_DDI_A_E,
2134         },
2135         {
2136                 .name = "DDI B IO power well",
2137                 .domains = SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2138                 .ops = &hsw_power_well_ops,
2139                 .id = SKL_DISP_PW_DDI_B,
2140         },
2141         {
2142                 .name = "DDI C IO power well",
2143                 .domains = SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2144                 .ops = &hsw_power_well_ops,
2145                 .id = SKL_DISP_PW_DDI_C,
2146         },
2147         {
2148                 .name = "DDI D IO power well",
2149                 .domains = SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS,
2150                 .ops = &hsw_power_well_ops,
2151                 .id = SKL_DISP_PW_DDI_D,
2152         },
2153 };
2154
2155 static struct i915_power_well bxt_power_wells[] = {
2156         {
2157                 .name = "always-on",
2158                 .always_on = 1,
2159                 .domains = POWER_DOMAIN_MASK,
2160                 .ops = &i9xx_always_on_power_well_ops,
2161                 .id = I915_DISP_PW_ALWAYS_ON,
2162         },
2163         {
2164                 .name = "power well 1",
2165                 .domains = 0,
2166                 .ops = &hsw_power_well_ops,
2167                 .id = SKL_DISP_PW_1,
2168                 {
2169                         .hsw.has_fuses = true,
2170                 },
2171         },
2172         {
2173                 .name = "DC off",
2174                 .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
2175                 .ops = &gen9_dc_off_power_well_ops,
2176                 .id = SKL_DISP_PW_DC_OFF,
2177         },
2178         {
2179                 .name = "power well 2",
2180                 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2181                 .ops = &hsw_power_well_ops,
2182                 .id = SKL_DISP_PW_2,
2183                 {
2184                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2185                         .hsw.has_vga = true,
2186                         .hsw.has_fuses = true,
2187                 },
2188         },
2189         {
2190                 .name = "dpio-common-a",
2191                 .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
2192                 .ops = &bxt_dpio_cmn_power_well_ops,
2193                 .id = BXT_DPIO_CMN_A,
2194                 {
2195                         .bxt.phy = DPIO_PHY1,
2196                 },
2197         },
2198         {
2199                 .name = "dpio-common-bc",
2200                 .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
2201                 .ops = &bxt_dpio_cmn_power_well_ops,
2202                 .id = BXT_DPIO_CMN_BC,
2203                 {
2204                         .bxt.phy = DPIO_PHY0,
2205                 },
2206         },
2207 };
2208
2209 static struct i915_power_well glk_power_wells[] = {
2210         {
2211                 .name = "always-on",
2212                 .always_on = 1,
2213                 .domains = POWER_DOMAIN_MASK,
2214                 .ops = &i9xx_always_on_power_well_ops,
2215                 .id = I915_DISP_PW_ALWAYS_ON,
2216         },
2217         {
2218                 .name = "power well 1",
2219                 /* Handled by the DMC firmware */
2220                 .domains = 0,
2221                 .ops = &hsw_power_well_ops,
2222                 .id = SKL_DISP_PW_1,
2223                 {
2224                         .hsw.has_fuses = true,
2225                 },
2226         },
2227         {
2228                 .name = "DC off",
2229                 .domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
2230                 .ops = &gen9_dc_off_power_well_ops,
2231                 .id = SKL_DISP_PW_DC_OFF,
2232         },
2233         {
2234                 .name = "power well 2",
2235                 .domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2236                 .ops = &hsw_power_well_ops,
2237                 .id = SKL_DISP_PW_2,
2238                 {
2239                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2240                         .hsw.has_vga = true,
2241                         .hsw.has_fuses = true,
2242                 },
2243         },
2244         {
2245                 .name = "dpio-common-a",
2246                 .domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
2247                 .ops = &bxt_dpio_cmn_power_well_ops,
2248                 .id = BXT_DPIO_CMN_A,
2249                 {
2250                         .bxt.phy = DPIO_PHY1,
2251                 },
2252         },
2253         {
2254                 .name = "dpio-common-b",
2255                 .domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
2256                 .ops = &bxt_dpio_cmn_power_well_ops,
2257                 .id = BXT_DPIO_CMN_BC,
2258                 {
2259                         .bxt.phy = DPIO_PHY0,
2260                 },
2261         },
2262         {
2263                 .name = "dpio-common-c",
2264                 .domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
2265                 .ops = &bxt_dpio_cmn_power_well_ops,
2266                 .id = GLK_DPIO_CMN_C,
2267                 {
2268                         .bxt.phy = DPIO_PHY2,
2269                 },
2270         },
2271         {
2272                 .name = "AUX A",
2273                 .domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
2274                 .ops = &hsw_power_well_ops,
2275                 .id = GLK_DISP_PW_AUX_A,
2276         },
2277         {
2278                 .name = "AUX B",
2279                 .domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
2280                 .ops = &hsw_power_well_ops,
2281                 .id = GLK_DISP_PW_AUX_B,
2282         },
2283         {
2284                 .name = "AUX C",
2285                 .domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
2286                 .ops = &hsw_power_well_ops,
2287                 .id = GLK_DISP_PW_AUX_C,
2288         },
2289         {
2290                 .name = "DDI A IO power well",
2291                 .domains = GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS,
2292                 .ops = &hsw_power_well_ops,
2293                 .id = GLK_DISP_PW_DDI_A,
2294         },
2295         {
2296                 .name = "DDI B IO power well",
2297                 .domains = GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2298                 .ops = &hsw_power_well_ops,
2299                 .id = SKL_DISP_PW_DDI_B,
2300         },
2301         {
2302                 .name = "DDI C IO power well",
2303                 .domains = GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2304                 .ops = &hsw_power_well_ops,
2305                 .id = SKL_DISP_PW_DDI_C,
2306         },
2307 };
2308
2309 static struct i915_power_well cnl_power_wells[] = {
2310         {
2311                 .name = "always-on",
2312                 .always_on = 1,
2313                 .domains = POWER_DOMAIN_MASK,
2314                 .ops = &i9xx_always_on_power_well_ops,
2315                 .id = I915_DISP_PW_ALWAYS_ON,
2316         },
2317         {
2318                 .name = "power well 1",
2319                 /* Handled by the DMC firmware */
2320                 .domains = 0,
2321                 .ops = &hsw_power_well_ops,
2322                 .id = SKL_DISP_PW_1,
2323                 {
2324                         .hsw.has_fuses = true,
2325                 },
2326         },
2327         {
2328                 .name = "AUX A",
2329                 .domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS,
2330                 .ops = &hsw_power_well_ops,
2331                 .id = CNL_DISP_PW_AUX_A,
2332         },
2333         {
2334                 .name = "AUX B",
2335                 .domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS,
2336                 .ops = &hsw_power_well_ops,
2337                 .id = CNL_DISP_PW_AUX_B,
2338         },
2339         {
2340                 .name = "AUX C",
2341                 .domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS,
2342                 .ops = &hsw_power_well_ops,
2343                 .id = CNL_DISP_PW_AUX_C,
2344         },
2345         {
2346                 .name = "AUX D",
2347                 .domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS,
2348                 .ops = &hsw_power_well_ops,
2349                 .id = CNL_DISP_PW_AUX_D,
2350         },
2351         {
2352                 .name = "DC off",
2353                 .domains = CNL_DISPLAY_DC_OFF_POWER_DOMAINS,
2354                 .ops = &gen9_dc_off_power_well_ops,
2355                 .id = SKL_DISP_PW_DC_OFF,
2356         },
2357         {
2358                 .name = "power well 2",
2359                 .domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2360                 .ops = &hsw_power_well_ops,
2361                 .id = SKL_DISP_PW_2,
2362                 {
2363                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2364                         .hsw.has_vga = true,
2365                         .hsw.has_fuses = true,
2366                 },
2367         },
2368         {
2369                 .name = "DDI A IO power well",
2370                 .domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS,
2371                 .ops = &hsw_power_well_ops,
2372                 .id = CNL_DISP_PW_DDI_A,
2373         },
2374         {
2375                 .name = "DDI B IO power well",
2376                 .domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS,
2377                 .ops = &hsw_power_well_ops,
2378                 .id = SKL_DISP_PW_DDI_B,
2379         },
2380         {
2381                 .name = "DDI C IO power well",
2382                 .domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS,
2383                 .ops = &hsw_power_well_ops,
2384                 .id = SKL_DISP_PW_DDI_C,
2385         },
2386         {
2387                 .name = "DDI D IO power well",
2388                 .domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS,
2389                 .ops = &hsw_power_well_ops,
2390                 .id = SKL_DISP_PW_DDI_D,
2391         },
2392 };
2393
2394 static int
2395 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
2396                                    int disable_power_well)
2397 {
2398         if (disable_power_well >= 0)
2399                 return !!disable_power_well;
2400
2401         return 1;
2402 }
2403
2404 static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
2405                                     int enable_dc)
2406 {
2407         uint32_t mask;
2408         int requested_dc;
2409         int max_dc;
2410
2411         if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
2412                 max_dc = 2;
2413                 mask = 0;
2414         } else if (IS_GEN9_LP(dev_priv)) {
2415                 max_dc = 1;
2416                 /*
2417                  * DC9 has a separate HW flow from the rest of the DC states,
2418                  * not depending on the DMC firmware. It's needed by system
2419                  * suspend/resume, so allow it unconditionally.
2420                  */
2421                 mask = DC_STATE_EN_DC9;
2422         } else {
2423                 max_dc = 0;
2424                 mask = 0;
2425         }
2426
2427         if (!i915.disable_power_well)
2428                 max_dc = 0;
2429
2430         if (enable_dc >= 0 && enable_dc <= max_dc) {
2431                 requested_dc = enable_dc;
2432         } else if (enable_dc == -1) {
2433                 requested_dc = max_dc;
2434         } else if (enable_dc > max_dc && enable_dc <= 2) {
2435                 DRM_DEBUG_KMS("Adjusting requested max DC state (%d->%d)\n",
2436                               enable_dc, max_dc);
2437                 requested_dc = max_dc;
2438         } else {
2439                 DRM_ERROR("Unexpected value for enable_dc (%d)\n", enable_dc);
2440                 requested_dc = max_dc;
2441         }
2442
2443         if (requested_dc > 1)
2444                 mask |= DC_STATE_EN_UPTO_DC6;
2445         if (requested_dc > 0)
2446                 mask |= DC_STATE_EN_UPTO_DC5;
2447
2448         DRM_DEBUG_KMS("Allowed DC state mask %02x\n", mask);
2449
2450         return mask;
2451 }
2452
2453 static void assert_power_well_ids_unique(struct drm_i915_private *dev_priv)
2454 {
2455         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2456         u64 power_well_ids;
2457         int i;
2458
2459         power_well_ids = 0;
2460         for (i = 0; i < power_domains->power_well_count; i++) {
2461                 enum i915_power_well_id id = power_domains->power_wells[i].id;
2462
2463                 WARN_ON(id >= sizeof(power_well_ids) * 8);
2464                 WARN_ON(power_well_ids & BIT_ULL(id));
2465                 power_well_ids |= BIT_ULL(id);
2466         }
2467 }
2468
2469 #define set_power_wells(power_domains, __power_wells) ({                \
2470         (power_domains)->power_wells = (__power_wells);                 \
2471         (power_domains)->power_well_count = ARRAY_SIZE(__power_wells);  \
2472 })
2473
2474 /**
2475  * intel_power_domains_init - initializes the power domain structures
2476  * @dev_priv: i915 device instance
2477  *
2478  * Initializes the power domain structures for @dev_priv depending upon the
2479  * supported platform.
2480  */
2481 int intel_power_domains_init(struct drm_i915_private *dev_priv)
2482 {
2483         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2484
2485         i915.disable_power_well = sanitize_disable_power_well_option(dev_priv,
2486                                                      i915.disable_power_well);
2487         dev_priv->csr.allowed_dc_mask = get_allowed_dc_mask(dev_priv,
2488                                                             i915.enable_dc);
2489
2490         BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
2491
2492         mutex_init(&power_domains->lock);
2493
2494         /*
2495          * The enabling order will be from lower to higher indexed wells,
2496          * the disabling order is reversed.
2497          */
2498         if (IS_HASWELL(dev_priv)) {
2499                 set_power_wells(power_domains, hsw_power_wells);
2500         } else if (IS_BROADWELL(dev_priv)) {
2501                 set_power_wells(power_domains, bdw_power_wells);
2502         } else if (IS_GEN9_BC(dev_priv)) {
2503                 set_power_wells(power_domains, skl_power_wells);
2504         } else if (IS_CANNONLAKE(dev_priv)) {
2505                 set_power_wells(power_domains, cnl_power_wells);
2506         } else if (IS_BROXTON(dev_priv)) {
2507                 set_power_wells(power_domains, bxt_power_wells);
2508         } else if (IS_GEMINILAKE(dev_priv)) {
2509                 set_power_wells(power_domains, glk_power_wells);
2510         } else if (IS_CHERRYVIEW(dev_priv)) {
2511                 set_power_wells(power_domains, chv_power_wells);
2512         } else if (IS_VALLEYVIEW(dev_priv)) {
2513                 set_power_wells(power_domains, vlv_power_wells);
2514         } else if (IS_I830(dev_priv)) {
2515                 set_power_wells(power_domains, i830_power_wells);
2516         } else {
2517                 set_power_wells(power_domains, i9xx_always_on_power_well);
2518         }
2519
2520         assert_power_well_ids_unique(dev_priv);
2521
2522         return 0;
2523 }
2524
2525 /**
2526  * intel_power_domains_fini - finalizes the power domain structures
2527  * @dev_priv: i915 device instance
2528  *
2529  * Finalizes the power domain structures for @dev_priv depending upon the
2530  * supported platform. This function also disables runtime pm and ensures that
2531  * the device stays powered up so that the driver can be reloaded.
2532  */
2533 void intel_power_domains_fini(struct drm_i915_private *dev_priv)
2534 {
2535         struct device *kdev = &dev_priv->drm.pdev->dev;
2536
2537         /*
2538          * The i915.ko module is still not prepared to be loaded when
2539          * the power well is not enabled, so just enable it in case
2540          * we're going to unload/reload.
2541          * The following also reacquires the RPM reference the core passed
2542          * to the driver during loading, which is dropped in
2543          * intel_runtime_pm_enable(). We have to hand back the control of the
2544          * device to the core with this reference held.
2545          */
2546         intel_display_set_init_power(dev_priv, true);
2547
2548         /* Remove the refcount we took to keep power well support disabled. */
2549         if (!i915.disable_power_well)
2550                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
2551
2552         /*
2553          * Remove the refcount we took in intel_runtime_pm_enable() in case
2554          * the platform doesn't support runtime PM.
2555          */
2556         if (!HAS_RUNTIME_PM(dev_priv))
2557                 pm_runtime_put(kdev);
2558 }
2559
2560 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
2561 {
2562         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2563         struct i915_power_well *power_well;
2564
2565         mutex_lock(&power_domains->lock);
2566         for_each_power_well(dev_priv, power_well) {
2567                 power_well->ops->sync_hw(dev_priv, power_well);
2568                 power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
2569                                                                      power_well);
2570         }
2571         mutex_unlock(&power_domains->lock);
2572 }
2573
2574 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
2575 {
2576         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
2577         POSTING_READ(DBUF_CTL);
2578
2579         udelay(10);
2580
2581         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
2582                 DRM_ERROR("DBuf power enable timeout\n");
2583 }
2584
2585 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
2586 {
2587         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
2588         POSTING_READ(DBUF_CTL);
2589
2590         udelay(10);
2591
2592         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
2593                 DRM_ERROR("DBuf power disable timeout!\n");
2594 }
2595
2596 static void skl_display_core_init(struct drm_i915_private *dev_priv,
2597                                    bool resume)
2598 {
2599         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2600         struct i915_power_well *well;
2601         uint32_t val;
2602
2603         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2604
2605         /* enable PCH reset handshake */
2606         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2607         I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
2608
2609         /* enable PG1 and Misc I/O */
2610         mutex_lock(&power_domains->lock);
2611
2612         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2613         intel_power_well_enable(dev_priv, well);
2614
2615         well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
2616         intel_power_well_enable(dev_priv, well);
2617
2618         mutex_unlock(&power_domains->lock);
2619
2620         skl_init_cdclk(dev_priv);
2621
2622         gen9_dbuf_enable(dev_priv);
2623
2624         if (resume && dev_priv->csr.dmc_payload)
2625                 intel_csr_load_program(dev_priv);
2626 }
2627
2628 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
2629 {
2630         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2631         struct i915_power_well *well;
2632
2633         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2634
2635         gen9_dbuf_disable(dev_priv);
2636
2637         skl_uninit_cdclk(dev_priv);
2638
2639         /* The spec doesn't call for removing the reset handshake flag */
2640         /* disable PG1 and Misc I/O */
2641
2642         mutex_lock(&power_domains->lock);
2643
2644         /*
2645          * BSpec says to keep the MISC IO power well enabled here, only
2646          * remove our request for power well 1.
2647          * Note that even though the driver's request is removed power well 1
2648          * may stay enabled after this due to DMC's own request on it.
2649          */
2650         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2651         intel_power_well_disable(dev_priv, well);
2652
2653         mutex_unlock(&power_domains->lock);
2654
2655         usleep_range(10, 30);           /* 10 us delay per Bspec */
2656 }
2657
2658 void bxt_display_core_init(struct drm_i915_private *dev_priv,
2659                            bool resume)
2660 {
2661         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2662         struct i915_power_well *well;
2663         uint32_t val;
2664
2665         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2666
2667         /*
2668          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
2669          * or else the reset will hang because there is no PCH to respond.
2670          * Move the handshake programming to initialization sequence.
2671          * Previously was left up to BIOS.
2672          */
2673         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2674         val &= ~RESET_PCH_HANDSHAKE_ENABLE;
2675         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2676
2677         /* Enable PG1 */
2678         mutex_lock(&power_domains->lock);
2679
2680         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2681         intel_power_well_enable(dev_priv, well);
2682
2683         mutex_unlock(&power_domains->lock);
2684
2685         bxt_init_cdclk(dev_priv);
2686
2687         gen9_dbuf_enable(dev_priv);
2688
2689         if (resume && dev_priv->csr.dmc_payload)
2690                 intel_csr_load_program(dev_priv);
2691 }
2692
2693 void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
2694 {
2695         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2696         struct i915_power_well *well;
2697
2698         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2699
2700         gen9_dbuf_disable(dev_priv);
2701
2702         bxt_uninit_cdclk(dev_priv);
2703
2704         /* The spec doesn't call for removing the reset handshake flag */
2705
2706         /*
2707          * Disable PW1 (PG1).
2708          * Note that even though the driver's request is removed power well 1
2709          * may stay enabled after this due to DMC's own request on it.
2710          */
2711         mutex_lock(&power_domains->lock);
2712
2713         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2714         intel_power_well_disable(dev_priv, well);
2715
2716         mutex_unlock(&power_domains->lock);
2717
2718         usleep_range(10, 30);           /* 10 us delay per Bspec */
2719 }
2720
2721 #define CNL_PROCMON_IDX(val) \
2722         (((val) & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) >> VOLTAGE_INFO_SHIFT)
2723 #define NUM_CNL_PROCMON \
2724         (CNL_PROCMON_IDX(VOLTAGE_INFO_MASK | PROCESS_INFO_MASK) + 1)
2725
2726 static const struct cnl_procmon {
2727         u32 dw1, dw9, dw10;
2728 } cnl_procmon_values[NUM_CNL_PROCMON] = {
2729         [CNL_PROCMON_IDX(VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0)] =
2730                 { .dw1 = 0x00 << 16, .dw9 = 0x62AB67BB, .dw10 = 0x51914F96, },
2731         [CNL_PROCMON_IDX(VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_0)] =
2732                 { .dw1 = 0x00 << 16, .dw9 = 0x86E172C7, .dw10 = 0x77CA5EAB, },
2733         [CNL_PROCMON_IDX(VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_1)] =
2734                 { .dw1 = 0x00 << 16, .dw9 = 0x93F87FE1, .dw10 = 0x8AE871C5, },
2735         [CNL_PROCMON_IDX(VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_0)] =
2736                 { .dw1 = 0x00 << 16, .dw9 = 0x98FA82DD, .dw10 = 0x89E46DC1, },
2737         [CNL_PROCMON_IDX(VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_1)] =
2738                 { .dw1 = 0x44 << 16, .dw9 = 0x9A00AB25, .dw10 = 0x8AE38FF1, },
2739 };
2740
2741 static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume)
2742 {
2743         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2744         const struct cnl_procmon *procmon;
2745         struct i915_power_well *well;
2746         u32 val;
2747
2748         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2749
2750         /* 1. Enable PCH Reset Handshake */
2751         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2752         val |= RESET_PCH_HANDSHAKE_ENABLE;
2753         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2754
2755         /* 2. Enable Comp */
2756         val = I915_READ(CHICKEN_MISC_2);
2757         val &= ~CNL_COMP_PWR_DOWN;
2758         I915_WRITE(CHICKEN_MISC_2, val);
2759
2760         val = I915_READ(CNL_PORT_COMP_DW3);
2761         procmon = &cnl_procmon_values[CNL_PROCMON_IDX(val)];
2762
2763         WARN_ON(procmon->dw10 == 0);
2764
2765         val = I915_READ(CNL_PORT_COMP_DW1);
2766         val &= ~((0xff << 16) | 0xff);
2767         val |= procmon->dw1;
2768         I915_WRITE(CNL_PORT_COMP_DW1, val);
2769
2770         I915_WRITE(CNL_PORT_COMP_DW9, procmon->dw9);
2771         I915_WRITE(CNL_PORT_COMP_DW10, procmon->dw10);
2772
2773         val = I915_READ(CNL_PORT_COMP_DW0);
2774         val |= COMP_INIT;
2775         I915_WRITE(CNL_PORT_COMP_DW0, val);
2776
2777         /* 3. */
2778         val = I915_READ(CNL_PORT_CL1CM_DW5);
2779         val |= CL_POWER_DOWN_ENABLE;
2780         I915_WRITE(CNL_PORT_CL1CM_DW5, val);
2781
2782         /*
2783          * 4. Enable Power Well 1 (PG1).
2784          *    The AUX IO power wells will be enabled on demand.
2785          */
2786         mutex_lock(&power_domains->lock);
2787         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2788         intel_power_well_enable(dev_priv, well);
2789         mutex_unlock(&power_domains->lock);
2790
2791         /* 5. Enable CD clock */
2792         cnl_init_cdclk(dev_priv);
2793
2794         /* 6. Enable DBUF */
2795         gen9_dbuf_enable(dev_priv);
2796
2797         if (resume && dev_priv->csr.dmc_payload)
2798                 intel_csr_load_program(dev_priv);
2799 }
2800
2801 #undef CNL_PROCMON_IDX
2802 #undef NUM_CNL_PROCMON
2803
2804 static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
2805 {
2806         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2807         struct i915_power_well *well;
2808         u32 val;
2809
2810         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2811
2812         /* 1. Disable all display engine functions -> aready done */
2813
2814         /* 2. Disable DBUF */
2815         gen9_dbuf_disable(dev_priv);
2816
2817         /* 3. Disable CD clock */
2818         cnl_uninit_cdclk(dev_priv);
2819
2820         /*
2821          * 4. Disable Power Well 1 (PG1).
2822          *    The AUX IO power wells are toggled on demand, so they are already
2823          *    disabled at this point.
2824          */
2825         mutex_lock(&power_domains->lock);
2826         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2827         intel_power_well_disable(dev_priv, well);
2828         mutex_unlock(&power_domains->lock);
2829
2830         usleep_range(10, 30);           /* 10 us delay per Bspec */
2831
2832         /* 5. Disable Comp */
2833         val = I915_READ(CHICKEN_MISC_2);
2834         val |= CNL_COMP_PWR_DOWN;
2835         I915_WRITE(CHICKEN_MISC_2, val);
2836 }
2837
2838 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
2839 {
2840         struct i915_power_well *cmn_bc =
2841                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2842         struct i915_power_well *cmn_d =
2843                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
2844
2845         /*
2846          * DISPLAY_PHY_CONTROL can get corrupted if read. As a
2847          * workaround never ever read DISPLAY_PHY_CONTROL, and
2848          * instead maintain a shadow copy ourselves. Use the actual
2849          * power well state and lane status to reconstruct the
2850          * expected initial value.
2851          */
2852         dev_priv->chv_phy_control =
2853                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
2854                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
2855                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
2856                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
2857                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
2858
2859         /*
2860          * If all lanes are disabled we leave the override disabled
2861          * with all power down bits cleared to match the state we
2862          * would use after disabling the port. Otherwise enable the
2863          * override and set the lane powerdown bits accding to the
2864          * current lane status.
2865          */
2866         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
2867                 uint32_t status = I915_READ(DPLL(PIPE_A));
2868                 unsigned int mask;
2869
2870                 mask = status & DPLL_PORTB_READY_MASK;
2871                 if (mask == 0xf)
2872                         mask = 0x0;
2873                 else
2874                         dev_priv->chv_phy_control |=
2875                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
2876
2877                 dev_priv->chv_phy_control |=
2878                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
2879
2880                 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
2881                 if (mask == 0xf)
2882                         mask = 0x0;
2883                 else
2884                         dev_priv->chv_phy_control |=
2885                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
2886
2887                 dev_priv->chv_phy_control |=
2888                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
2889
2890                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
2891
2892                 dev_priv->chv_phy_assert[DPIO_PHY0] = false;
2893         } else {
2894                 dev_priv->chv_phy_assert[DPIO_PHY0] = true;
2895         }
2896
2897         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
2898                 uint32_t status = I915_READ(DPIO_PHY_STATUS);
2899                 unsigned int mask;
2900
2901                 mask = status & DPLL_PORTD_READY_MASK;
2902
2903                 if (mask == 0xf)
2904                         mask = 0x0;
2905                 else
2906                         dev_priv->chv_phy_control |=
2907                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
2908
2909                 dev_priv->chv_phy_control |=
2910                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
2911
2912                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
2913
2914                 dev_priv->chv_phy_assert[DPIO_PHY1] = false;
2915         } else {
2916                 dev_priv->chv_phy_assert[DPIO_PHY1] = true;
2917         }
2918
2919         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
2920
2921         DRM_DEBUG_KMS("Initial PHY_CONTROL=0x%08x\n",
2922                       dev_priv->chv_phy_control);
2923 }
2924
2925 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
2926 {
2927         struct i915_power_well *cmn =
2928                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2929         struct i915_power_well *disp2d =
2930                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);
2931
2932         /* If the display might be already active skip this */
2933         if (cmn->ops->is_enabled(dev_priv, cmn) &&
2934             disp2d->ops->is_enabled(dev_priv, disp2d) &&
2935             I915_READ(DPIO_CTL) & DPIO_CMNRST)
2936                 return;
2937
2938         DRM_DEBUG_KMS("toggling display PHY side reset\n");
2939
2940         /* cmnlane needs DPLL registers */
2941         disp2d->ops->enable(dev_priv, disp2d);
2942
2943         /*
2944          * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
2945          * Need to assert and de-assert PHY SB reset by gating the
2946          * common lane power, then un-gating it.
2947          * Simply ungating isn't enough to reset the PHY enough to get
2948          * ports and lanes running.
2949          */
2950         cmn->ops->disable(dev_priv, cmn);
2951 }
2952
2953 /**
2954  * intel_power_domains_init_hw - initialize hardware power domain state
2955  * @dev_priv: i915 device instance
2956  * @resume: Called from resume code paths or not
2957  *
2958  * This function initializes the hardware power domain state and enables all
2959  * power wells belonging to the INIT power domain. Power wells in other
2960  * domains (and not in the INIT domain) are referenced or disabled during the
2961  * modeset state HW readout. After that the reference count of each power well
2962  * must match its HW enabled state, see intel_power_domains_verify_state().
2963  */
2964 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
2965 {
2966         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2967
2968         power_domains->initializing = true;
2969
2970         if (IS_CANNONLAKE(dev_priv)) {
2971                 cnl_display_core_init(dev_priv, resume);
2972         } else if (IS_GEN9_BC(dev_priv)) {
2973                 skl_display_core_init(dev_priv, resume);
2974         } else if (IS_GEN9_LP(dev_priv)) {
2975                 bxt_display_core_init(dev_priv, resume);
2976         } else if (IS_CHERRYVIEW(dev_priv)) {
2977                 mutex_lock(&power_domains->lock);
2978                 chv_phy_control_init(dev_priv);
2979                 mutex_unlock(&power_domains->lock);
2980         } else if (IS_VALLEYVIEW(dev_priv)) {
2981                 mutex_lock(&power_domains->lock);
2982                 vlv_cmnlane_wa(dev_priv);
2983                 mutex_unlock(&power_domains->lock);
2984         }
2985
2986         /* For now, we need the power well to be always enabled. */
2987         intel_display_set_init_power(dev_priv, true);
2988         /* Disable power support if the user asked so. */
2989         if (!i915.disable_power_well)
2990                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
2991         intel_power_domains_sync_hw(dev_priv);
2992         power_domains->initializing = false;
2993 }
2994
2995 /**
2996  * intel_power_domains_suspend - suspend power domain state
2997  * @dev_priv: i915 device instance
2998  *
2999  * This function prepares the hardware power domain state before entering
3000  * system suspend. It must be paired with intel_power_domains_init_hw().
3001  */
3002 void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
3003 {
3004         /*
3005          * Even if power well support was disabled we still want to disable
3006          * power wells while we are system suspended.
3007          */
3008         if (!i915.disable_power_well)
3009                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
3010
3011         if (IS_CANNONLAKE(dev_priv))
3012                 cnl_display_core_uninit(dev_priv);
3013         else if (IS_GEN9_BC(dev_priv))
3014                 skl_display_core_uninit(dev_priv);
3015         else if (IS_GEN9_LP(dev_priv))
3016                 bxt_display_core_uninit(dev_priv);
3017 }
3018
3019 static void intel_power_domains_dump_info(struct drm_i915_private *dev_priv)
3020 {
3021         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3022         struct i915_power_well *power_well;
3023
3024         for_each_power_well(dev_priv, power_well) {
3025                 enum intel_display_power_domain domain;
3026
3027                 DRM_DEBUG_DRIVER("%-25s %d\n",
3028                                  power_well->name, power_well->count);
3029
3030                 for_each_power_domain(domain, power_well->domains)
3031                         DRM_DEBUG_DRIVER("  %-23s %d\n",
3032                                          intel_display_power_domain_str(domain),
3033                                          power_domains->domain_use_count[domain]);
3034         }
3035 }
3036
3037 /**
3038  * intel_power_domains_verify_state - verify the HW/SW state for all power wells
3039  * @dev_priv: i915 device instance
3040  *
3041  * Verify if the reference count of each power well matches its HW enabled
3042  * state and the total refcount of the domains it belongs to. This must be
3043  * called after modeset HW state sanitization, which is responsible for
3044  * acquiring reference counts for any power wells in use and disabling the
3045  * ones left on by BIOS but not required by any active output.
3046  */
3047 void intel_power_domains_verify_state(struct drm_i915_private *dev_priv)
3048 {
3049         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3050         struct i915_power_well *power_well;
3051         bool dump_domain_info;
3052
3053         mutex_lock(&power_domains->lock);
3054
3055         dump_domain_info = false;
3056         for_each_power_well(dev_priv, power_well) {
3057                 enum intel_display_power_domain domain;
3058                 int domains_count;
3059                 bool enabled;
3060
3061                 /*
3062                  * Power wells not belonging to any domain (like the MISC_IO
3063                  * and PW1 power wells) are under FW control, so ignore them,
3064                  * since their state can change asynchronously.
3065                  */
3066                 if (!power_well->domains)
3067                         continue;
3068
3069                 enabled = power_well->ops->is_enabled(dev_priv, power_well);
3070                 if ((power_well->count || power_well->always_on) != enabled)
3071                         DRM_ERROR("power well %s state mismatch (refcount %d/enabled %d)",
3072                                   power_well->name, power_well->count, enabled);
3073
3074                 domains_count = 0;
3075                 for_each_power_domain(domain, power_well->domains)
3076                         domains_count += power_domains->domain_use_count[domain];
3077
3078                 if (power_well->count != domains_count) {
3079                         DRM_ERROR("power well %s refcount/domain refcount mismatch "
3080                                   "(refcount %d/domains refcount %d)\n",
3081                                   power_well->name, power_well->count,
3082                                   domains_count);
3083                         dump_domain_info = true;
3084                 }
3085         }
3086
3087         if (dump_domain_info) {
3088                 static bool dumped;
3089
3090                 if (!dumped) {
3091                         intel_power_domains_dump_info(dev_priv);
3092                         dumped = true;
3093                 }
3094         }
3095
3096         mutex_unlock(&power_domains->lock);
3097 }
3098
3099 /**
3100  * intel_runtime_pm_get - grab a runtime pm reference
3101  * @dev_priv: i915 device instance
3102  *
3103  * This function grabs a device-level runtime pm reference (mostly used for GEM
3104  * code to ensure the GTT or GT is on) and ensures that it is powered up.
3105  *
3106  * Any runtime pm reference obtained by this function must have a symmetric
3107  * call to intel_runtime_pm_put() to release the reference again.
3108  */
3109 void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
3110 {
3111         struct pci_dev *pdev = dev_priv->drm.pdev;
3112         struct device *kdev = &pdev->dev;
3113         int ret;
3114
3115         ret = pm_runtime_get_sync(kdev);
3116         WARN_ONCE(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
3117
3118         atomic_inc(&dev_priv->pm.wakeref_count);
3119         assert_rpm_wakelock_held(dev_priv);
3120 }
3121
3122 /**
3123  * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use
3124  * @dev_priv: i915 device instance
3125  *
3126  * This function grabs a device-level runtime pm reference if the device is
3127  * already in use and ensures that it is powered up.
3128  *
3129  * Any runtime pm reference obtained by this function must have a symmetric
3130  * call to intel_runtime_pm_put() to release the reference again.
3131  */
3132 bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
3133 {
3134         struct pci_dev *pdev = dev_priv->drm.pdev;
3135         struct device *kdev = &pdev->dev;
3136
3137         if (IS_ENABLED(CONFIG_PM)) {
3138                 int ret = pm_runtime_get_if_in_use(kdev);
3139
3140                 /*
3141                  * In cases runtime PM is disabled by the RPM core and we get
3142                  * an -EINVAL return value we are not supposed to call this
3143                  * function, since the power state is undefined. This applies
3144                  * atm to the late/early system suspend/resume handlers.
3145                  */
3146                 WARN_ONCE(ret < 0,
3147                           "pm_runtime_get_if_in_use() failed: %d\n", ret);
3148                 if (ret <= 0)
3149                         return false;
3150         }
3151
3152         atomic_inc(&dev_priv->pm.wakeref_count);
3153         assert_rpm_wakelock_held(dev_priv);
3154
3155         return true;
3156 }
3157
3158 /**
3159  * intel_runtime_pm_get_noresume - grab a runtime pm reference
3160  * @dev_priv: i915 device instance
3161  *
3162  * This function grabs a device-level runtime pm reference (mostly used for GEM
3163  * code to ensure the GTT or GT is on).
3164  *
3165  * It will _not_ power up the device but instead only check that it's powered
3166  * on.  Therefore it is only valid to call this functions from contexts where
3167  * the device is known to be powered up and where trying to power it up would
3168  * result in hilarity and deadlocks. That pretty much means only the system
3169  * suspend/resume code where this is used to grab runtime pm references for
3170  * delayed setup down in work items.
3171  *
3172  * Any runtime pm reference obtained by this function must have a symmetric
3173  * call to intel_runtime_pm_put() to release the reference again.
3174  */
3175 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
3176 {
3177         struct pci_dev *pdev = dev_priv->drm.pdev;
3178         struct device *kdev = &pdev->dev;
3179
3180         assert_rpm_wakelock_held(dev_priv);
3181         pm_runtime_get_noresume(kdev);
3182
3183         atomic_inc(&dev_priv->pm.wakeref_count);
3184 }
3185
3186 /**
3187  * intel_runtime_pm_put - release a runtime pm reference
3188  * @dev_priv: i915 device instance
3189  *
3190  * This function drops the device-level runtime pm reference obtained by
3191  * intel_runtime_pm_get() and might power down the corresponding
3192  * hardware block right away if this is the last reference.
3193  */
3194 void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
3195 {
3196         struct pci_dev *pdev = dev_priv->drm.pdev;
3197         struct device *kdev = &pdev->dev;
3198
3199         assert_rpm_wakelock_held(dev_priv);
3200         atomic_dec(&dev_priv->pm.wakeref_count);
3201
3202         pm_runtime_mark_last_busy(kdev);
3203         pm_runtime_put_autosuspend(kdev);
3204 }
3205
3206 /**
3207  * intel_runtime_pm_enable - enable runtime pm
3208  * @dev_priv: i915 device instance
3209  *
3210  * This function enables runtime pm at the end of the driver load sequence.
3211  *
3212  * Note that this function does currently not enable runtime pm for the
3213  * subordinate display power domains. That is only done on the first modeset
3214  * using intel_display_set_init_power().
3215  */
3216 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
3217 {
3218         struct pci_dev *pdev = dev_priv->drm.pdev;
3219         struct device *kdev = &pdev->dev;
3220
3221         pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
3222         pm_runtime_mark_last_busy(kdev);
3223
3224         /*
3225          * Take a permanent reference to disable the RPM functionality and drop
3226          * it only when unloading the driver. Use the low level get/put helpers,
3227          * so the driver's own RPM reference tracking asserts also work on
3228          * platforms without RPM support.
3229          */
3230         if (!HAS_RUNTIME_PM(dev_priv)) {
3231                 int ret;
3232
3233                 pm_runtime_dont_use_autosuspend(kdev);
3234                 ret = pm_runtime_get_sync(kdev);
3235                 WARN(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
3236         } else {
3237                 pm_runtime_use_autosuspend(kdev);
3238         }
3239
3240         /*
3241          * The core calls the driver load handler with an RPM reference held.
3242          * We drop that here and will reacquire it during unloading in
3243          * intel_power_domains_fini().
3244          */
3245         pm_runtime_put_autosuspend(kdev);
3246 }