drivers/gpu/drm/i915/gt/intel_workarounds.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2018 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8 #include "intel_context.h"
   9 #include "intel_engine_pm.h"
  10 #include "intel_gt.h"
  11 #include "intel_ring.h"
  12 #include "intel_workarounds.h"
  13
  14 /**
  15  * DOC: Hardware workarounds
  16  *
  17  * This file is intended as a central place to implement most [1]_ of the
  18  * required workarounds for hardware to work as originally intended. They fall
  19  * in five basic categories depending on how/when they are applied:
  20  *
  21  * - Workarounds that touch registers that are saved/restored to/from the HW
  22  *   context image. The list is emitted (via Load Register Immediate commands)
  23  *   everytime a new context is created.
  24  * - GT workarounds. The list of these WAs is applied whenever these registers
  25  *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
  26  * - Display workarounds. The list is applied during display clock-gating
  27  *   initialization.
  28  * - Workarounds that whitelist a privileged register, so that UMDs can manage
  29  *   them directly. This is just a special case of a MMMIO workaround (as we
  30  *   write the list of these to/be-whitelisted registers to some special HW
  31  *   registers).
  32  * - Workaround batchbuffers, that get executed automatically by the hardware
  33  *   on every HW context restore.
  34  *
  35  * .. [1] Please notice that there are other WAs that, due to their nature,
  36  *    cannot be applied from a central place. Those are peppered around the rest
  37  *    of the code, as needed.
  38  *
  39  * .. [2] Technically, some registers are powercontext saved & restored, so they
  40  *    survive a suspend/resume. In practice, writing them again is not too
  41  *    costly and simplifies things. We can revisit this in the future.
  42  *
  43  * Layout
  44  * ~~~~~~
  45  *
  46  * Keep things in this file ordered by WA type, as per the above (context, GT,
  47  * display, register whitelist, batchbuffer). Then, inside each type, keep the
  48  * following order:
  49  *
  50  * - Infrastructure functions and macros
  51  * - WAs per platform in standard gen/chrono order
  52  * - Public functions to init or apply the given workaround type.
  53  */
  54
  55 /*
  56  * KBL revision ID ordering is bizarre; higher revision ID's map to lower
  57  * steppings in some cases.  So rather than test against the revision ID
  58  * directly, let's map that into our own range of increasing ID's that we
  59  * can test against in a regular manner.
  60  */
  61
  62 const struct i915_rev_steppings kbl_revids[] = {
  63         [0] = { .gt_stepping = KBL_REVID_A0, .disp_stepping = KBL_REVID_A0 },
  64         [1] = { .gt_stepping = KBL_REVID_B0, .disp_stepping = KBL_REVID_B0 },
  65         [2] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B0 },
  66         [3] = { .gt_stepping = KBL_REVID_D0, .disp_stepping = KBL_REVID_B0 },
  67         [4] = { .gt_stepping = KBL_REVID_F0, .disp_stepping = KBL_REVID_C0 },
  68         [5] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B1 },
  69         [6] = { .gt_stepping = KBL_REVID_D1, .disp_stepping = KBL_REVID_B1 },
  70         [7] = { .gt_stepping = KBL_REVID_G0, .disp_stepping = KBL_REVID_C0 },
  71 };
  72
  73 const struct i915_rev_steppings tgl_uy_revids[] = {
  74         [0] = { .gt_stepping = TGL_REVID_A0, .disp_stepping = TGL_REVID_A0 },
  75         [1] = { .gt_stepping = TGL_REVID_B0, .disp_stepping = TGL_REVID_C0 },
  76         [2] = { .gt_stepping = TGL_REVID_B1, .disp_stepping = TGL_REVID_C0 },
  77         [3] = { .gt_stepping = TGL_REVID_C0, .disp_stepping = TGL_REVID_D0 },
  78 };
  79
  80 /* Same GT stepping between tgl_uy_revids and tgl_revids don't mean the same HW */
  81 const struct i915_rev_steppings tgl_revids[] = {
  82         [0] = { .gt_stepping = TGL_REVID_A0, .disp_stepping = TGL_REVID_B0 },
  83         [1] = { .gt_stepping = TGL_REVID_B0, .disp_stepping = TGL_REVID_D0 },
  84 };
  85
  86 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
  87 {
  88         wal->name = name;
  89         wal->engine_name = engine_name;
  90 }
  91
  92 #define WA_LIST_CHUNK (1 << 4)
  93
  94 static void wa_init_finish(struct i915_wa_list *wal)
  95 {
  96         /* Trim unused entries. */
  97         if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
  98                 struct i915_wa *list = kmemdup(wal->list,
  99                                                wal->count * sizeof(*list),
 100                                                GFP_KERNEL);
 101
 102                 if (list) {
 103                         kfree(wal->list);
 104                         wal->list = list;
 105                 }
 106         }
 107
 108         if (!wal->count)
 109                 return;
 110
 111         DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
 112                          wal->wa_count, wal->name, wal->engine_name);
 113 }
 114
 115 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
 116 {
 117         unsigned int addr = i915_mmio_reg_offset(wa->reg);
 118         unsigned int start = 0, end = wal->count;
 119         const unsigned int grow = WA_LIST_CHUNK;
 120         struct i915_wa *wa_;
 121
 122         GEM_BUG_ON(!is_power_of_2(grow));
 123
 124         if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
 125                 struct i915_wa *list;
 126
 127                 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
 128                                      GFP_KERNEL);
 129                 if (!list) {
 130                         DRM_ERROR("No space for workaround init!\n");
 131                         return;
 132                 }
 133
 134                 if (wal->list) {
 135                         memcpy(list, wal->list, sizeof(*wa) * wal->count);
 136                         kfree(wal->list);
 137                 }
 138
 139                 wal->list = list;
 140         }
 141
 142         while (start < end) {
 143                 unsigned int mid = start + (end - start) / 2;
 144
 145                 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
 146                         start = mid + 1;
 147                 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
 148                         end = mid;
 149                 } else {
 150                         wa_ = &wal->list[mid];
 151
 152                         if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
 153                                 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
 154                                           i915_mmio_reg_offset(wa_->reg),
 155                                           wa_->clr, wa_->set);
 156
 157                                 wa_->set &= ~wa->clr;
 158                         }
 159
 160                         wal->wa_count++;
 161                         wa_->set |= wa->set;
 162                         wa_->clr |= wa->clr;
 163                         wa_->read |= wa->read;
 164                         return;
 165                 }
 166         }
 167
 168         wal->wa_count++;
 169         wa_ = &wal->list[wal->count++];
 170         *wa_ = *wa;
 171
 172         while (wa_-- > wal->list) {
 173                 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
 174                            i915_mmio_reg_offset(wa_[1].reg));
 175                 if (i915_mmio_reg_offset(wa_[1].reg) >
 176                     i915_mmio_reg_offset(wa_[0].reg))
 177                         break;
 178
 179                 swap(wa_[1], wa_[0]);
 180         }
 181 }
 182
 183 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
 184                    u32 clear, u32 set, u32 read_mask)
 185 {
 186         struct i915_wa wa = {
 187                 .reg  = reg,
 188                 .clr  = clear,
 189                 .set  = set,
 190                 .read = read_mask,
 191         };
 192
 193         _wa_add(wal, &wa);
 194 }
 195
 196 static void
 197 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
 198 {
 199         wa_add(wal, reg, clear, set, clear);
 200 }
 201
 202 static void
 203 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 204 {
 205         wa_write_masked_or(wal, reg, ~0, set);
 206 }
 207
 208 static void
 209 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 210 {
 211         wa_write_masked_or(wal, reg, set, set);
 212 }
 213
 214 static void
 215 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
 216 {
 217         wa_write_masked_or(wal, reg, clr, 0);
 218 }
 219
 220 static void
 221 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 222 {
 223         wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val);
 224 }
 225
 226 static void
 227 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 228 {
 229         wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val);
 230 }
 231
 232 #define WA_SET_BIT_MASKED(addr, mask) \
 233         wa_masked_en(wal, (addr), (mask))
 234
 235 #define WA_CLR_BIT_MASKED(addr, mask) \
 236         wa_masked_dis(wal, (addr), (mask))
 237
 238 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 239         wa_write_masked_or(wal, (addr), 0, _MASKED_FIELD((mask), (value)))
 240
 241 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
 242                                       struct i915_wa_list *wal)
 243 {
 244         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 245 }
 246
 247 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
 248                                       struct i915_wa_list *wal)
 249 {
 250         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 251 }
 252
 253 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
 254                                       struct i915_wa_list *wal)
 255 {
 256         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 257
 258         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 259         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 260
 261         /* WaDisablePartialInstShootdown:bdw,chv */
 262         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 263                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 264
 265         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 266          * workaround for for a possible hang in the unlikely event a TLB
 267          * invalidation occurs during a PSD flush.
 268          */
 269         /* WaForceEnableNonCoherent:bdw,chv */
 270         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 271         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 272                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 273                           HDC_FORCE_NON_COHERENT);
 274
 275         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 276          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 277          *  polygons in the same 8x4 pixel/sample area to be processed without
 278          *  stalling waiting for the earlier ones to write to Hierarchical Z
 279          *  buffer."
 280          *
 281          * This optimization is off by default for BDW and CHV; turn it on.
 282          */
 283         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 284
 285         /* Wa4x4STCOptimizationDisable:bdw,chv */
 286         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 287
 288         /*
 289          * BSpec recommends 8x4 when MSAA is used,
 290          * however in practice 16x4 seems fastest.
 291          *
 292          * Note that PS/WM thread counts depend on the WIZ hashing
 293          * disable bit, which we don't touch here, but it's good
 294          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 295          */
 296         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 297                             GEN6_WIZ_HASHING_MASK,
 298                             GEN6_WIZ_HASHING_16x4);
 299 }
 300
 301 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
 302                                      struct i915_wa_list *wal)
 303 {
 304         struct drm_i915_private *i915 = engine->i915;
 305
 306         gen8_ctx_workarounds_init(engine, wal);
 307
 308         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 309         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 310
 311         /* WaDisableDopClockGating:bdw
 312          *
 313          * Also see the related UCGTCL1 write in bdw_init_clock_gating()
 314          * to disable EUTC clock gating.
 315          */
 316         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 317                           DOP_CLOCK_GATING_DISABLE);
 318
 319         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 320                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 321
 322         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 323                           /* WaForceContextSaveRestoreNonCoherent:bdw */
 324                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 325                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 326                           (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 327 }
 328
 329 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
 330                                      struct i915_wa_list *wal)
 331 {
 332         gen8_ctx_workarounds_init(engine, wal);
 333
 334         /* WaDisableThreadStallDopClockGating:chv */
 335         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 336
 337         /* Improve HiZ throughput on CHV. */
 338         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 339 }
 340
 341 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
 342                                       struct i915_wa_list *wal)
 343 {
 344         struct drm_i915_private *i915 = engine->i915;
 345
 346         if (HAS_LLC(i915)) {
 347                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 348                  *
 349                  * Must match Display Engine. See
 350                  * WaCompressedResourceDisplayNewHashMode.
 351                  */
 352                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 353                                   GEN9_PBE_COMPRESSED_HASH_SELECTION);
 354                 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 355                                   GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
 356         }
 357
 358         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
 359         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
 360         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 361                           FLOW_CONTROL_ENABLE |
 362                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 363
 364         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
 365         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 366         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 367                           GEN9_ENABLE_YV12_BUGFIX |
 368                           GEN9_ENABLE_GPGPU_PREEMPTION);
 369
 370         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
 371         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
 372         WA_SET_BIT_MASKED(CACHE_MODE_1,
 373                           GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 374                           GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
 375
 376         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
 377         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 378                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 379
 380         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 381         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 382                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 383                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 384
 385         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 386          * both tied to WaForceContextSaveRestoreNonCoherent
 387          * in some hsds for skl. We keep the tie for all gen9. The
 388          * documentation is a bit hazy and so we want to get common behaviour,
 389          * even though there is no clear evidence we would need both on kbl/bxt.
 390          * This area has been source of system hangs so we play it safe
 391          * and mimic the skl regardless of what bspec says.
 392          *
 393          * Use Force Non-Coherent whenever executing a 3D context. This
 394          * is a workaround for a possible hang in the unlikely event
 395          * a TLB invalidation occurs during a PSD flush.
 396          */
 397
 398         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
 399         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 400                           HDC_FORCE_NON_COHERENT);
 401
 402         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 403         if (IS_SKYLAKE(i915) ||
 404             IS_KABYLAKE(i915) ||
 405             IS_COFFEELAKE(i915) ||
 406             IS_COMETLAKE(i915))
 407                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 408                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
 409
 410         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 411         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 412
 413         /*
 414          * Supporting preemption with fine-granularity requires changes in the
 415          * batch buffer programming. Since we can't break old userspace, we
 416          * need to set our default preemption level to safe value. Userspace is
 417          * still able to use more fine-grained preemption levels, since in
 418          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
 419          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
 420          * not real HW workarounds, but merely a way to start using preemption
 421          * while maintaining old contract with userspace.
 422          */
 423
 424         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
 425         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 426
 427         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
 428         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 429                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 430                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 431
 432         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
 433         if (IS_GEN9_LP(i915))
 434                 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
 435 }
 436
 437 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
 438                                 struct i915_wa_list *wal)
 439 {
 440         struct intel_gt *gt = engine->gt;
 441         u8 vals[3] = { 0, 0, 0 };
 442         unsigned int i;
 443
 444         for (i = 0; i < 3; i++) {
 445                 u8 ss;
 446
 447                 /*
 448                  * Only consider slices where one, and only one, subslice has 7
 449                  * EUs
 450                  */
 451                 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i]))
 452                         continue;
 453
 454                 /*
 455                  * subslice_7eu[i] != 0 (because of the check above) and
 456                  * ss_max == 4 (maximum number of subslices possible per slice)
 457                  *
 458                  * ->    0 <= ss <= 3;
 459                  */
 460                 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
 461                 vals[i] = 3 - ss;
 462         }
 463
 464         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 465                 return;
 466
 467         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 468         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 469                             GEN9_IZ_HASHING_MASK(2) |
 470                             GEN9_IZ_HASHING_MASK(1) |
 471                             GEN9_IZ_HASHING_MASK(0),
 472                             GEN9_IZ_HASHING(2, vals[2]) |
 473                             GEN9_IZ_HASHING(1, vals[1]) |
 474                             GEN9_IZ_HASHING(0, vals[0]));
 475 }
 476
 477 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
 478                                      struct i915_wa_list *wal)
 479 {
 480         gen9_ctx_workarounds_init(engine, wal);
 481         skl_tune_iz_hashing(engine, wal);
 482 }
 483
 484 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
 485                                      struct i915_wa_list *wal)
 486 {
 487         gen9_ctx_workarounds_init(engine, wal);
 488
 489         /* WaDisableThreadStallDopClockGating:bxt */
 490         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 491                           STALL_DOP_GATING_DISABLE);
 492
 493         /* WaToEnableHwFixForPushConstHWBug:bxt */
 494         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 495                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 496 }
 497
 498 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
 499                                      struct i915_wa_list *wal)
 500 {
 501         struct drm_i915_private *i915 = engine->i915;
 502
 503         gen9_ctx_workarounds_init(engine, wal);
 504
 505         /* WaToEnableHwFixForPushConstHWBug:kbl */
 506         if (IS_KBL_GT_REVID(i915, KBL_REVID_C0, REVID_FOREVER))
 507                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 508                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 509
 510         /* WaDisableSbeCacheDispatchPortSharing:kbl */
 511         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 512                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 513 }
 514
 515 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
 516                                      struct i915_wa_list *wal)
 517 {
 518         gen9_ctx_workarounds_init(engine, wal);
 519
 520         /* WaToEnableHwFixForPushConstHWBug:glk */
 521         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 522                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 523 }
 524
 525 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
 526                                      struct i915_wa_list *wal)
 527 {
 528         gen9_ctx_workarounds_init(engine, wal);
 529
 530         /* WaToEnableHwFixForPushConstHWBug:cfl */
 531         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 532                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 533
 534         /* WaDisableSbeCacheDispatchPortSharing:cfl */
 535         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 536                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 537 }
 538
 539 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
 540                                      struct i915_wa_list *wal)
 541 {
 542         /* WaForceContextSaveRestoreNonCoherent:cnl */
 543         WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
 544                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
 545
 546         /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
 547         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 548                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 549
 550         /* WaPushConstantDereferenceHoldDisable:cnl */
 551         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
 552
 553         /* FtrEnableFastAnisoL1BankingFix:cnl */
 554         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
 555
 556         /* WaDisable3DMidCmdPreemption:cnl */
 557         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 558
 559         /* WaDisableGPGPUMidCmdPreemption:cnl */
 560         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 561                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 562                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 563
 564         /* WaDisableEarlyEOT:cnl */
 565         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
 566 }
 567
 568 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
 569                                      struct i915_wa_list *wal)
 570 {
 571         struct drm_i915_private *i915 = engine->i915;
 572
 573         /* WaDisableBankHangMode:icl */
 574         wa_write(wal,
 575                  GEN8_L3CNTLREG,
 576                  intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
 577                  GEN8_ERRDETBCTRL);
 578
 579         /* Wa_1604370585:icl (pre-prod)
 580          * Formerly known as WaPushConstantDereferenceHoldDisable
 581          */
 582         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 583                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 584                                   PUSH_CONSTANT_DEREF_DISABLE);
 585
 586         /* WaForceEnableNonCoherent:icl
 587          * This is not the same workaround as in early Gen9 platforms, where
 588          * lacking this could cause system hangs, but coherency performance
 589          * overhead is high and only a few compute workloads really need it
 590          * (the register is whitelisted in hardware now, so UMDs can opt in
 591          * for coherency if they have a good reason).
 592          */
 593         WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
 594
 595         /* Wa_2006611047:icl (pre-prod)
 596          * Formerly known as WaDisableImprovedTdlClkGating
 597          */
 598         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 599                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 600                                   GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
 601
 602         /* Wa_2006665173:icl (pre-prod) */
 603         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 604                 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 605                                   GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
 606
 607         /* WaEnableFloatBlendOptimization:icl */
 608         wa_write_masked_or(wal,
 609                            GEN10_CACHE_MODE_SS,
 610                            0, /* write-only, so skip validation */
 611                            _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
 612
 613         /* WaDisableGPGPUMidThreadPreemption:icl */
 614         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 615                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 616                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 617
 618         /* allow headerless messages for preemptible GPGPU context */
 619         WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE,
 620                           GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
 621
 622         /* Wa_1604278689:icl,ehl */
 623         wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
 624         wa_write_masked_or(wal, IVB_FBC_RT_BASE_UPPER,
 625                            0, /* write-only register; skip validation */
 626                            0xFFFFFFFF);
 627
 628         /* Wa_1406306137:icl,ehl */
 629         wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
 630 }
 631
 632 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
 633                                        struct i915_wa_list *wal)
 634 {
 635         /*
 636          * Wa_1409142259:tgl
 637          * Wa_1409347922:tgl
 638          * Wa_1409252684:tgl
 639          * Wa_1409217633:tgl
 640          * Wa_1409207793:tgl
 641          * Wa_1409178076:tgl
 642          * Wa_1408979724:tgl
 643          * Wa_14010443199:rkl
 644          * Wa_14010698770:rkl
 645          */
 646         WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 647                           GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
 648
 649         /* WaDisableGPGPUMidThreadPreemption:gen12 */
 650         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 651                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 652                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 653 }
 654
 655 static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine,
 656                                      struct i915_wa_list *wal)
 657 {
 658         gen12_ctx_workarounds_init(engine, wal);
 659
 660         /*
 661          * Wa_1604555607:tgl,rkl
 662          *
 663          * Note that the implementation of this workaround is further modified
 664          * according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
 665          * FF_MODE2 register will return the wrong value when read. The default
 666          * value for this register is zero for all fields and there are no bit
 667          * masks. So instead of doing a RMW we should just write the GS Timer
 668          * and TDS timer values for Wa_1604555607 and Wa_16011163337.
 669          */
 670         wa_add(wal,
 671                FF_MODE2,
 672                FF_MODE2_GS_TIMER_MASK | FF_MODE2_TDS_TIMER_MASK,
 673                FF_MODE2_GS_TIMER_224  | FF_MODE2_TDS_TIMER_128,
 674                0);
 675 }
 676
 677 static void
 678 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
 679                            struct i915_wa_list *wal,
 680                            const char *name)
 681 {
 682         struct drm_i915_private *i915 = engine->i915;
 683
 684         if (engine->class != RENDER_CLASS)
 685                 return;
 686
 687         wa_init_start(wal, name, engine->name);
 688
 689         if (IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915))
 690                 tgl_ctx_workarounds_init(engine, wal);
 691         else if (IS_GEN(i915, 12))
 692                 gen12_ctx_workarounds_init(engine, wal);
 693         else if (IS_GEN(i915, 11))
 694                 icl_ctx_workarounds_init(engine, wal);
 695         else if (IS_CANNONLAKE(i915))
 696                 cnl_ctx_workarounds_init(engine, wal);
 697         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
 698                 cfl_ctx_workarounds_init(engine, wal);
 699         else if (IS_GEMINILAKE(i915))
 700                 glk_ctx_workarounds_init(engine, wal);
 701         else if (IS_KABYLAKE(i915))
 702                 kbl_ctx_workarounds_init(engine, wal);
 703         else if (IS_BROXTON(i915))
 704                 bxt_ctx_workarounds_init(engine, wal);
 705         else if (IS_SKYLAKE(i915))
 706                 skl_ctx_workarounds_init(engine, wal);
 707         else if (IS_CHERRYVIEW(i915))
 708                 chv_ctx_workarounds_init(engine, wal);
 709         else if (IS_BROADWELL(i915))
 710                 bdw_ctx_workarounds_init(engine, wal);
 711         else if (IS_GEN(i915, 7))
 712                 gen7_ctx_workarounds_init(engine, wal);
 713         else if (IS_GEN(i915, 6))
 714                 gen6_ctx_workarounds_init(engine, wal);
 715         else if (INTEL_GEN(i915) < 8)
 716                 return;
 717         else
 718                 MISSING_CASE(INTEL_GEN(i915));
 719
 720         wa_init_finish(wal);
 721 }
 722
 723 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
 724 {
 725         __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
 726 }
 727
 728 int intel_engine_emit_ctx_wa(struct i915_request *rq)
 729 {
 730         struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
 731         struct i915_wa *wa;
 732         unsigned int i;
 733         u32 *cs;
 734         int ret;
 735
 736         if (wal->count == 0)
 737                 return 0;
 738
 739         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 740         if (ret)
 741                 return ret;
 742
 743         cs = intel_ring_begin(rq, (wal->count * 2 + 2));
 744         if (IS_ERR(cs))
 745                 return PTR_ERR(cs);
 746
 747         *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
 748         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 749                 *cs++ = i915_mmio_reg_offset(wa->reg);
 750                 *cs++ = wa->set;
 751         }
 752         *cs++ = MI_NOOP;
 753
 754         intel_ring_advance(rq, cs);
 755
 756         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 757         if (ret)
 758                 return ret;
 759
 760         return 0;
 761 }
 762
 763 static void
 764 gen4_gt_workarounds_init(struct drm_i915_private *i915,
 765                          struct i915_wa_list *wal)
 766 {
 767         /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
 768         wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
 769 }
 770
 771 static void
 772 g4x_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 773 {
 774         gen4_gt_workarounds_init(i915, wal);
 775
 776         /* WaDisableRenderCachePipelinedFlush:g4x,ilk */
 777         wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
 778 }
 779
 780 static void
 781 ilk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 782 {
 783         g4x_gt_workarounds_init(i915, wal);
 784
 785         wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
 786 }
 787
 788 static void
 789 snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 790 {
 791         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
 792         wa_masked_en(wal,
 793                      _3D_CHICKEN,
 794                      _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
 795
 796         /* WaDisable_RenderCache_OperationalFlush:snb */
 797         wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
 798
 799         /*
 800          * BSpec recommends 8x4 when MSAA is used,
 801          * however in practice 16x4 seems fastest.
 802          *
 803          * Note that PS/WM thread counts depend on the WIZ hashing
 804          * disable bit, which we don't touch here, but it's good
 805          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 806          */
 807         wa_add(wal,
 808                GEN6_GT_MODE, 0,
 809                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 810                GEN6_WIZ_HASHING_16x4);
 811
 812         wa_masked_dis(wal, CACHE_MODE_0, CM0_STC_EVICT_DISABLE_LRA_SNB);
 813
 814         wa_masked_en(wal,
 815                      _3D_CHICKEN3,
 816                      /* WaStripsFansDisableFastClipPerformanceFix:snb */
 817                      _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
 818                      /*
 819                       * Bspec says:
 820                       * "This bit must be set if 3DSTATE_CLIP clip mode is set
 821                       * to normal and 3DSTATE_SF number of SF output attributes
 822                       * is more than 16."
 823                       */
 824                    _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
 825 }
 826
 827 static void
 828 ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 829 {
 830         /* WaDisableEarlyCull:ivb */
 831         wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
 832
 833         /* WaDisablePSDDualDispatchEnable:ivb */
 834         if (IS_IVB_GT1(i915))
 835                 wa_masked_en(wal,
 836                              GEN7_HALF_SLICE_CHICKEN1,
 837                              GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
 838
 839         /* WaDisable_RenderCache_OperationalFlush:ivb */
 840         wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
 841
 842         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
 843         wa_masked_dis(wal,
 844                       GEN7_COMMON_SLICE_CHICKEN1,
 845                       GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
 846
 847         /* WaApplyL3ControlAndL3ChickenMode:ivb */
 848         wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
 849         wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
 850
 851         /* WaForceL3Serialization:ivb */
 852         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
 853
 854         /*
 855          * WaVSThreadDispatchOverride:ivb,vlv
 856          *
 857          * This actually overrides the dispatch
 858          * mode for all thread types.
 859          */
 860         wa_write_masked_or(wal, GEN7_FF_THREAD_MODE,
 861                            GEN7_FF_SCHED_MASK,
 862                            GEN7_FF_TS_SCHED_HW |
 863                            GEN7_FF_VS_SCHED_HW |
 864                            GEN7_FF_DS_SCHED_HW);
 865
 866         if (0) { /* causes HiZ corruption on ivb:gt1 */
 867                 /* enable HiZ Raw Stall Optimization */
 868                 wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 869         }
 870
 871         /* WaDisable4x2SubspanOptimization:ivb */
 872         wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
 873
 874         /*
 875          * BSpec recommends 8x4 when MSAA is used,
 876          * however in practice 16x4 seems fastest.
 877          *
 878          * Note that PS/WM thread counts depend on the WIZ hashing
 879          * disable bit, which we don't touch here, but it's good
 880          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 881          */
 882         wa_add(wal, GEN7_GT_MODE, 0,
 883                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 884                GEN6_WIZ_HASHING_16x4);
 885 }
 886
 887 static void
 888 vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 889 {
 890         /* WaDisableEarlyCull:vlv */
 891         wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
 892
 893         /* WaPsdDispatchEnable:vlv */
 894         /* WaDisablePSDDualDispatchEnable:vlv */
 895         wa_masked_en(wal,
 896                      GEN7_HALF_SLICE_CHICKEN1,
 897                      GEN7_MAX_PS_THREAD_DEP |
 898                      GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
 899
 900         /* WaDisable_RenderCache_OperationalFlush:vlv */
 901         wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
 902
 903         /* WaForceL3Serialization:vlv */
 904         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
 905
 906         /*
 907          * WaVSThreadDispatchOverride:ivb,vlv
 908          *
 909          * This actually overrides the dispatch
 910          * mode for all thread types.
 911          */
 912         wa_write_masked_or(wal,
 913                            GEN7_FF_THREAD_MODE,
 914                            GEN7_FF_SCHED_MASK,
 915                            GEN7_FF_TS_SCHED_HW |
 916                            GEN7_FF_VS_SCHED_HW |
 917                            GEN7_FF_DS_SCHED_HW);
 918
 919         /*
 920          * BSpec says this must be set, even though
 921          * WaDisable4x2SubspanOptimization isn't listed for VLV.
 922          */
 923         wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
 924
 925         /*
 926          * BSpec recommends 8x4 when MSAA is used,
 927          * however in practice 16x4 seems fastest.
 928          *
 929          * Note that PS/WM thread counts depend on the WIZ hashing
 930          * disable bit, which we don't touch here, but it's good
 931          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 932          */
 933         wa_add(wal, GEN7_GT_MODE, 0,
 934                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 935                GEN6_WIZ_HASHING_16x4);
 936
 937         /*
 938          * WaIncreaseL3CreditsForVLVB0:vlv
 939          * This is the hardware default actually.
 940          */
 941         wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
 942 }
 943
 944 static void
 945 hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 946 {
 947         /* L3 caching of data atomics doesn't work -- disable it. */
 948         wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
 949
 950         wa_add(wal,
 951                HSW_ROW_CHICKEN3, 0,
 952                _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
 953                 0 /* XXX does this reg exist? */);
 954
 955         /* WaVSRefCountFullforceMissDisable:hsw */
 956         wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
 957
 958         wa_masked_dis(wal,
 959                       CACHE_MODE_0_GEN7,
 960                       /* WaDisable_RenderCache_OperationalFlush:hsw */
 961                       RC_OP_FLUSH_ENABLE |
 962                       /* enable HiZ Raw Stall Optimization */
 963                       HIZ_RAW_STALL_OPT_DISABLE);
 964
 965         /* WaDisable4x2SubspanOptimization:hsw */
 966         wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
 967
 968         /*
 969          * BSpec recommends 8x4 when MSAA is used,
 970          * however in practice 16x4 seems fastest.
 971          *
 972          * Note that PS/WM thread counts depend on the WIZ hashing
 973          * disable bit, which we don't touch here, but it's good
 974          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 975          */
 976         wa_add(wal, GEN7_GT_MODE, 0,
 977                _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
 978                GEN6_WIZ_HASHING_16x4);
 979
 980         /* WaSampleCChickenBitEnable:hsw */
 981         wa_masked_en(wal, HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
 982 }
 983
 984 static void
 985 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 986 {
 987         /* WaDisableKillLogic:bxt,skl,kbl */
 988         if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
 989                 wa_write_or(wal,
 990                             GAM_ECOCHK,
 991                             ECOCHK_DIS_TLB);
 992
 993         if (HAS_LLC(i915)) {
 994                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 995                  *
 996                  * Must match Display Engine. See
 997                  * WaCompressedResourceDisplayNewHashMode.
 998                  */
 999                 wa_write_or(wal,
1000                             MMCD_MISC_CTRL,
1001                             MMCD_PCLA | MMCD_HOTSPOT_EN);
1002         }
1003
1004         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1005         wa_write_or(wal,
1006                     GAM_ECOCHK,
1007                     BDW_DISABLE_HDC_INVALIDATION);
1008 }
1009
1010 static void
1011 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1012 {
1013         gen9_gt_workarounds_init(i915, wal);
1014
1015         /* WaDisableGafsUnitClkGating:skl */
1016         wa_write_or(wal,
1017                     GEN7_UCGCTL4,
1018                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1019
1020         /* WaInPlaceDecompressionHang:skl */
1021         if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
1022                 wa_write_or(wal,
1023                             GEN9_GAMT_ECO_REG_RW_IA,
1024                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1025 }
1026
1027 static void
1028 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1029 {
1030         gen9_gt_workarounds_init(i915, wal);
1031
1032         /* WaInPlaceDecompressionHang:bxt */
1033         wa_write_or(wal,
1034                     GEN9_GAMT_ECO_REG_RW_IA,
1035                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1036 }
1037
1038 static void
1039 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1040 {
1041         gen9_gt_workarounds_init(i915, wal);
1042
1043         /* WaDisableDynamicCreditSharing:kbl */
1044         if (IS_KBL_GT_REVID(i915, 0, KBL_REVID_B0))
1045                 wa_write_or(wal,
1046                             GAMT_CHKN_BIT_REG,
1047                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
1048
1049         /* WaDisableGafsUnitClkGating:kbl */
1050         wa_write_or(wal,
1051                     GEN7_UCGCTL4,
1052                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1053
1054         /* WaInPlaceDecompressionHang:kbl */
1055         wa_write_or(wal,
1056                     GEN9_GAMT_ECO_REG_RW_IA,
1057                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1058 }
1059
1060 static void
1061 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1062 {
1063         gen9_gt_workarounds_init(i915, wal);
1064 }
1065
1066 static void
1067 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1068 {
1069         gen9_gt_workarounds_init(i915, wal);
1070
1071         /* WaDisableGafsUnitClkGating:cfl */
1072         wa_write_or(wal,
1073                     GEN7_UCGCTL4,
1074                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1075
1076         /* WaInPlaceDecompressionHang:cfl */
1077         wa_write_or(wal,
1078                     GEN9_GAMT_ECO_REG_RW_IA,
1079                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1080 }
1081
1082 static void
1083 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
1084 {
1085         const struct sseu_dev_info *sseu = &i915->gt.info.sseu;
1086         unsigned int slice, subslice;
1087         u32 l3_en, mcr, mcr_mask;
1088
1089         GEM_BUG_ON(INTEL_GEN(i915) < 10);
1090
1091         /*
1092          * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
1093          * L3Banks could be fused off in single slice scenario. If that is
1094          * the case, we might need to program MCR select to a valid L3Bank
1095          * by default, to make sure we correctly read certain registers
1096          * later on (in the range 0xB100 - 0xB3FF).
1097          *
1098          * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
1099          * Before any MMIO read into slice/subslice specific registers, MCR
1100          * packet control register needs to be programmed to point to any
1101          * enabled s/ss pair. Otherwise, incorrect values will be returned.
1102          * This means each subsequent MMIO read will be forwarded to an
1103          * specific s/ss combination, but this is OK since these registers
1104          * are consistent across s/ss in almost all cases. In the rare
1105          * occasions, such as INSTDONE, where this value is dependent
1106          * on s/ss combo, the read should be done with read_subslice_reg.
1107          *
1108          * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both
1109          * to which subslice, or to which L3 bank, the respective mmio reads
1110          * will go, we have to find a common index which works for both
1111          * accesses.
1112          *
1113          * Case where we cannot find a common index fortunately should not
1114          * happen in production hardware, so we only emit a warning instead of
1115          * implementing something more complex that requires checking the range
1116          * of every MMIO read.
1117          */
1118
1119         if (INTEL_GEN(i915) >= 10 && is_power_of_2(sseu->slice_mask)) {
1120                 u32 l3_fuse =
1121                         intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) &
1122                         GEN10_L3BANK_MASK;
1123
1124                 drm_dbg(&i915->drm, "L3 fuse = %x\n", l3_fuse);
1125                 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse);
1126         } else {
1127                 l3_en = ~0;
1128         }
1129
1130         slice = fls(sseu->slice_mask) - 1;
1131         subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice));
1132         if (!subslice) {
1133                 drm_warn(&i915->drm,
1134                          "No common index found between subslice mask %x and L3 bank mask %x!\n",
1135                          intel_sseu_get_subslices(sseu, slice), l3_en);
1136                 subslice = fls(l3_en);
1137                 drm_WARN_ON(&i915->drm, !subslice);
1138         }
1139         subslice--;
1140
1141         if (INTEL_GEN(i915) >= 11) {
1142                 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
1143                 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
1144         } else {
1145                 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
1146                 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
1147         }
1148
1149         drm_dbg(&i915->drm, "MCR slice/subslice = %x\n", mcr);
1150
1151         wa_write_masked_or(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
1152 }
1153
1154 static void
1155 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1156 {
1157         wa_init_mcr(i915, wal);
1158
1159         /* WaInPlaceDecompressionHang:cnl */
1160         wa_write_or(wal,
1161                     GEN9_GAMT_ECO_REG_RW_IA,
1162                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1163 }
1164
1165 static void
1166 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1167 {
1168         wa_init_mcr(i915, wal);
1169
1170         /* WaInPlaceDecompressionHang:icl */
1171         wa_write_or(wal,
1172                     GEN9_GAMT_ECO_REG_RW_IA,
1173                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1174
1175         /* WaModifyGamTlbPartitioning:icl */
1176         wa_write_masked_or(wal,
1177                            GEN11_GACB_PERF_CTRL,
1178                            GEN11_HASH_CTRL_MASK,
1179                            GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1180
1181         /* Wa_1405766107:icl
1182          * Formerly known as WaCL2SFHalfMaxAlloc
1183          */
1184         wa_write_or(wal,
1185                     GEN11_LSN_UNSLCVC,
1186                     GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1187                     GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1188
1189         /* Wa_220166154:icl
1190          * Formerly known as WaDisCtxReload
1191          */
1192         wa_write_or(wal,
1193                     GEN8_GAMW_ECO_DEV_RW_IA,
1194                     GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1195
1196         /* Wa_1405779004:icl (pre-prod) */
1197         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
1198                 wa_write_or(wal,
1199                             SLICE_UNIT_LEVEL_CLKGATE,
1200                             MSCUNIT_CLKGATE_DIS);
1201
1202         /* Wa_1406838659:icl (pre-prod) */
1203         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1204                 wa_write_or(wal,
1205                             INF_UNIT_LEVEL_CLKGATE,
1206                             CGPSF_CLKGATE_DIS);
1207
1208         /* Wa_1406463099:icl
1209          * Formerly known as WaGamTlbPendError
1210          */
1211         wa_write_or(wal,
1212                     GAMT_CHKN_BIT_REG,
1213                     GAMT_CHKN_DISABLE_L3_COH_PIPE);
1214
1215         /*
1216          * Wa_1408615072:icl,ehl  (vsunit)
1217          * Wa_1407596294:icl,ehl  (hsunit)
1218          */
1219         wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1220                     VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1221
1222         /* Wa_1407352427:icl,ehl */
1223         wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1224                     PSDUNIT_CLKGATE_DIS);
1225
1226         /* Wa_1406680159:icl,ehl */
1227         wa_write_or(wal,
1228                     SUBSLICE_UNIT_LEVEL_CLKGATE,
1229                     GWUNIT_CLKGATE_DIS);
1230
1231         /* Wa_1607087056:icl,ehl,jsl */
1232         if (IS_ICELAKE(i915) ||
1233             IS_EHL_REVID(i915, EHL_REVID_A0, EHL_REVID_A0)) {
1234                 wa_write_or(wal,
1235                             SLICE_UNIT_LEVEL_CLKGATE,
1236                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1237         }
1238 }
1239
1240 static void
1241 gen12_gt_workarounds_init(struct drm_i915_private *i915,
1242                           struct i915_wa_list *wal)
1243 {
1244         wa_init_mcr(i915, wal);
1245 }
1246
1247 static void
1248 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1249 {
1250         gen12_gt_workarounds_init(i915, wal);
1251
1252         /* Wa_1409420604:tgl */
1253         if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
1254                 wa_write_or(wal,
1255                             SUBSLICE_UNIT_LEVEL_CLKGATE2,
1256                             CPSSUNIT_CLKGATE_DIS);
1257
1258         /* Wa_1607087056:tgl also know as BUG:1409180338 */
1259         if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
1260                 wa_write_or(wal,
1261                             SLICE_UNIT_LEVEL_CLKGATE,
1262                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1263 }
1264
1265 static void
1266 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
1267 {
1268         if (IS_TIGERLAKE(i915))
1269                 tgl_gt_workarounds_init(i915, wal);
1270         else if (IS_GEN(i915, 12))
1271                 gen12_gt_workarounds_init(i915, wal);
1272         else if (IS_GEN(i915, 11))
1273                 icl_gt_workarounds_init(i915, wal);
1274         else if (IS_CANNONLAKE(i915))
1275                 cnl_gt_workarounds_init(i915, wal);
1276         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1277                 cfl_gt_workarounds_init(i915, wal);
1278         else if (IS_GEMINILAKE(i915))
1279                 glk_gt_workarounds_init(i915, wal);
1280         else if (IS_KABYLAKE(i915))
1281                 kbl_gt_workarounds_init(i915, wal);
1282         else if (IS_BROXTON(i915))
1283                 bxt_gt_workarounds_init(i915, wal);
1284         else if (IS_SKYLAKE(i915))
1285                 skl_gt_workarounds_init(i915, wal);
1286         else if (IS_HASWELL(i915))
1287                 hsw_gt_workarounds_init(i915, wal);
1288         else if (IS_VALLEYVIEW(i915))
1289                 vlv_gt_workarounds_init(i915, wal);
1290         else if (IS_IVYBRIDGE(i915))
1291                 ivb_gt_workarounds_init(i915, wal);
1292         else if (IS_GEN(i915, 6))
1293                 snb_gt_workarounds_init(i915, wal);
1294         else if (IS_GEN(i915, 5))
1295                 ilk_gt_workarounds_init(i915, wal);
1296         else if (IS_G4X(i915))
1297                 g4x_gt_workarounds_init(i915, wal);
1298         else if (IS_GEN(i915, 4))
1299                 gen4_gt_workarounds_init(i915, wal);
1300         else if (INTEL_GEN(i915) <= 8)
1301                 return;
1302         else
1303                 MISSING_CASE(INTEL_GEN(i915));
1304 }
1305
1306 void intel_gt_init_workarounds(struct drm_i915_private *i915)
1307 {
1308         struct i915_wa_list *wal = &i915->gt_wa_list;
1309
1310         wa_init_start(wal, "GT", "global");
1311         gt_init_workarounds(i915, wal);
1312         wa_init_finish(wal);
1313 }
1314
1315 static enum forcewake_domains
1316 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1317 {
1318         enum forcewake_domains fw = 0;
1319         struct i915_wa *wa;
1320         unsigned int i;
1321
1322         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1323                 fw |= intel_uncore_forcewake_for_reg(uncore,
1324                                                      wa->reg,
1325                                                      FW_REG_READ |
1326                                                      FW_REG_WRITE);
1327
1328         return fw;
1329 }
1330
1331 static bool
1332 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1333 {
1334         if ((cur ^ wa->set) & wa->read) {
1335                 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x)\n",
1336                           name, from, i915_mmio_reg_offset(wa->reg),
1337                           cur, cur & wa->read, wa->set);
1338
1339                 return false;
1340         }
1341
1342         return true;
1343 }
1344
1345 static void
1346 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1347 {
1348         enum forcewake_domains fw;
1349         unsigned long flags;
1350         struct i915_wa *wa;
1351         unsigned int i;
1352
1353         if (!wal->count)
1354                 return;
1355
1356         fw = wal_get_fw_for_rmw(uncore, wal);
1357
1358         spin_lock_irqsave(&uncore->lock, flags);
1359         intel_uncore_forcewake_get__locked(uncore, fw);
1360
1361         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1362                 if (wa->clr)
1363                         intel_uncore_rmw_fw(uncore, wa->reg, wa->clr, wa->set);
1364                 else
1365                         intel_uncore_write_fw(uncore, wa->reg, wa->set);
1366                 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1367                         wa_verify(wa,
1368                                   intel_uncore_read_fw(uncore, wa->reg),
1369                                   wal->name, "application");
1370         }
1371
1372         intel_uncore_forcewake_put__locked(uncore, fw);
1373         spin_unlock_irqrestore(&uncore->lock, flags);
1374 }
1375
1376 void intel_gt_apply_workarounds(struct intel_gt *gt)
1377 {
1378         wa_list_apply(gt->uncore, &gt->i915->gt_wa_list);
1379 }
1380
1381 static bool wa_list_verify(struct intel_uncore *uncore,
1382                            const struct i915_wa_list *wal,
1383                            const char *from)
1384 {
1385         struct i915_wa *wa;
1386         unsigned int i;
1387         bool ok = true;
1388
1389         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1390                 ok &= wa_verify(wa,
1391                                 intel_uncore_read(uncore, wa->reg),
1392                                 wal->name, from);
1393
1394         return ok;
1395 }
1396
1397 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1398 {
1399         return wa_list_verify(gt->uncore, &gt->i915->gt_wa_list, from);
1400 }
1401
1402 static inline bool is_nonpriv_flags_valid(u32 flags)
1403 {
1404         /* Check only valid flag bits are set */
1405         if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1406                 return false;
1407
1408         /* NB: Only 3 out of 4 enum values are valid for access field */
1409         if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1410             RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1411                 return false;
1412
1413         return true;
1414 }
1415
1416 static void
1417 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1418 {
1419         struct i915_wa wa = {
1420                 .reg = reg
1421         };
1422
1423         if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1424                 return;
1425
1426         if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1427                 return;
1428
1429         wa.reg.reg |= flags;
1430         _wa_add(wal, &wa);
1431 }
1432
1433 static void
1434 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1435 {
1436         whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1437 }
1438
1439 static void gen9_whitelist_build(struct i915_wa_list *w)
1440 {
1441         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1442         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1443
1444         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1445         whitelist_reg(w, GEN8_CS_CHICKEN1);
1446
1447         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1448         whitelist_reg(w, GEN8_HDC_CHICKEN1);
1449
1450         /* WaSendPushConstantsFromMMIO:skl,bxt */
1451         whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1452 }
1453
1454 static void skl_whitelist_build(struct intel_engine_cs *engine)
1455 {
1456         struct i915_wa_list *w = &engine->whitelist;
1457
1458         if (engine->class != RENDER_CLASS)
1459                 return;
1460
1461         gen9_whitelist_build(w);
1462
1463         /* WaDisableLSQCROPERFforOCL:skl */
1464         whitelist_reg(w, GEN8_L3SQCREG4);
1465 }
1466
1467 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1468 {
1469         if (engine->class != RENDER_CLASS)
1470                 return;
1471
1472         gen9_whitelist_build(&engine->whitelist);
1473 }
1474
1475 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1476 {
1477         struct i915_wa_list *w = &engine->whitelist;
1478
1479         if (engine->class != RENDER_CLASS)
1480                 return;
1481
1482         gen9_whitelist_build(w);
1483
1484         /* WaDisableLSQCROPERFforOCL:kbl */
1485         whitelist_reg(w, GEN8_L3SQCREG4);
1486 }
1487
1488 static void glk_whitelist_build(struct intel_engine_cs *engine)
1489 {
1490         struct i915_wa_list *w = &engine->whitelist;
1491
1492         if (engine->class != RENDER_CLASS)
1493                 return;
1494
1495         gen9_whitelist_build(w);
1496
1497         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1498         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1499 }
1500
1501 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1502 {
1503         struct i915_wa_list *w = &engine->whitelist;
1504
1505         if (engine->class != RENDER_CLASS)
1506                 return;
1507
1508         gen9_whitelist_build(w);
1509
1510         /*
1511          * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1512          *
1513          * This covers 4 register which are next to one another :
1514          *   - PS_INVOCATION_COUNT
1515          *   - PS_INVOCATION_COUNT_UDW
1516          *   - PS_DEPTH_COUNT
1517          *   - PS_DEPTH_COUNT_UDW
1518          */
1519         whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1520                           RING_FORCE_TO_NONPRIV_ACCESS_RD |
1521                           RING_FORCE_TO_NONPRIV_RANGE_4);
1522 }
1523
1524 static void cml_whitelist_build(struct intel_engine_cs *engine)
1525 {
1526         struct i915_wa_list *w = &engine->whitelist;
1527
1528         if (engine->class != RENDER_CLASS)
1529                 whitelist_reg_ext(w,
1530                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1531                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1532
1533         cfl_whitelist_build(engine);
1534 }
1535
1536 static void cnl_whitelist_build(struct intel_engine_cs *engine)
1537 {
1538         struct i915_wa_list *w = &engine->whitelist;
1539
1540         if (engine->class != RENDER_CLASS)
1541                 return;
1542
1543         /* WaEnablePreemptionGranularityControlByUMD:cnl */
1544         whitelist_reg(w, GEN8_CS_CHICKEN1);
1545 }
1546
1547 static void icl_whitelist_build(struct intel_engine_cs *engine)
1548 {
1549         struct i915_wa_list *w = &engine->whitelist;
1550
1551         switch (engine->class) {
1552         case RENDER_CLASS:
1553                 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1554                 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1555
1556                 /* WaAllowUMDToModifySamplerMode:icl */
1557                 whitelist_reg(w, GEN10_SAMPLER_MODE);
1558
1559                 /* WaEnableStateCacheRedirectToCS:icl */
1560                 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1561
1562                 /*
1563                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1564                  *
1565                  * This covers 4 register which are next to one another :
1566                  *   - PS_INVOCATION_COUNT
1567                  *   - PS_INVOCATION_COUNT_UDW
1568                  *   - PS_DEPTH_COUNT
1569                  *   - PS_DEPTH_COUNT_UDW
1570                  */
1571                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1572                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1573                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1574                 break;
1575
1576         case VIDEO_DECODE_CLASS:
1577                 /* hucStatusRegOffset */
1578                 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1579                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1580                 /* hucUKernelHdrInfoRegOffset */
1581                 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1582                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1583                 /* hucStatus2RegOffset */
1584                 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1585                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1586                 whitelist_reg_ext(w,
1587                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1588                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1589                 break;
1590
1591         default:
1592                 whitelist_reg_ext(w,
1593                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1594                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1595                 break;
1596         }
1597 }
1598
1599 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1600 {
1601         struct i915_wa_list *w = &engine->whitelist;
1602
1603         switch (engine->class) {
1604         case RENDER_CLASS:
1605                 /*
1606                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1607                  * Wa_1408556865:tgl
1608                  *
1609                  * This covers 4 registers which are next to one another :
1610                  *   - PS_INVOCATION_COUNT
1611                  *   - PS_INVOCATION_COUNT_UDW
1612                  *   - PS_DEPTH_COUNT
1613                  *   - PS_DEPTH_COUNT_UDW
1614                  */
1615                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1616                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1617                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1618
1619                 /* Wa_1808121037:tgl */
1620                 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1621
1622                 /* Wa_1806527549:tgl */
1623                 whitelist_reg(w, HIZ_CHICKEN);
1624                 break;
1625         default:
1626                 whitelist_reg_ext(w,
1627                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1628                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1629                 break;
1630         }
1631 }
1632
1633 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1634 {
1635         struct drm_i915_private *i915 = engine->i915;
1636         struct i915_wa_list *w = &engine->whitelist;
1637
1638         wa_init_start(w, "whitelist", engine->name);
1639
1640         if (IS_GEN(i915, 12))
1641                 tgl_whitelist_build(engine);
1642         else if (IS_GEN(i915, 11))
1643                 icl_whitelist_build(engine);
1644         else if (IS_CANNONLAKE(i915))
1645                 cnl_whitelist_build(engine);
1646         else if (IS_COMETLAKE(i915))
1647                 cml_whitelist_build(engine);
1648         else if (IS_COFFEELAKE(i915))
1649                 cfl_whitelist_build(engine);
1650         else if (IS_GEMINILAKE(i915))
1651                 glk_whitelist_build(engine);
1652         else if (IS_KABYLAKE(i915))
1653                 kbl_whitelist_build(engine);
1654         else if (IS_BROXTON(i915))
1655                 bxt_whitelist_build(engine);
1656         else if (IS_SKYLAKE(i915))
1657                 skl_whitelist_build(engine);
1658         else if (INTEL_GEN(i915) <= 8)
1659                 return;
1660         else
1661                 MISSING_CASE(INTEL_GEN(i915));
1662
1663         wa_init_finish(w);
1664 }
1665
1666 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1667 {
1668         const struct i915_wa_list *wal = &engine->whitelist;
1669         struct intel_uncore *uncore = engine->uncore;
1670         const u32 base = engine->mmio_base;
1671         struct i915_wa *wa;
1672         unsigned int i;
1673
1674         if (!wal->count)
1675                 return;
1676
1677         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1678                 intel_uncore_write(uncore,
1679                                    RING_FORCE_TO_NONPRIV(base, i),
1680                                    i915_mmio_reg_offset(wa->reg));
1681
1682         /* And clear the rest just in case of garbage */
1683         for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1684                 intel_uncore_write(uncore,
1685                                    RING_FORCE_TO_NONPRIV(base, i),
1686                                    i915_mmio_reg_offset(RING_NOPID(base)));
1687 }
1688
1689 static void
1690 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1691 {
1692         struct drm_i915_private *i915 = engine->i915;
1693
1694         if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) {
1695                 /*
1696                  * Wa_1607138336:tgl
1697                  * Wa_1607063988:tgl
1698                  */
1699                 wa_write_or(wal,
1700                             GEN9_CTX_PREEMPT_REG,
1701                             GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1702
1703                 /*
1704                  * Wa_1606679103:tgl
1705                  * (see also Wa_1606682166:icl)
1706                  */
1707                 wa_write_or(wal,
1708                             GEN7_SARCHKMD,
1709                             GEN7_DISABLE_SAMPLER_PREFETCH);
1710
1711                 /* Wa_1408615072:tgl */
1712                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1713                             VSUNIT_CLKGATE_DIS_TGL);
1714         }
1715
1716         if (IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1717                 /* Wa_1606931601:tgl,rkl */
1718                 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
1719
1720                 /* Wa_1409804808:tgl,rkl */
1721                 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
1722                              GEN12_PUSH_CONST_DEREF_HOLD_DIS);
1723
1724                 /*
1725                  * Wa_1409085225:tgl
1726                  * Wa_14010229206:tgl,rkl
1727                  */
1728                 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
1729
1730                 /*
1731                  * Wa_1407928979:tgl A*
1732                  * Wa_18011464164:tgl B0+
1733                  * Wa_22010931296:tgl B0+
1734                  * Wa_14010919138:rkl,tgl
1735                  */
1736                 wa_write_or(wal, GEN7_FF_THREAD_MODE,
1737                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1738
1739                 /*
1740                  * Wa_1607030317:tgl
1741                  * Wa_1607186500:tgl
1742                  * Wa_1607297627:tgl,rkl there are multiple entries for this
1743                  * WA in the BSpec; some indicate this is an A0-only WA,
1744                  * others indicate it applies to all steppings.
1745                  */
1746                 wa_masked_en(wal,
1747                              GEN6_RC_SLEEP_PSMI_CONTROL,
1748                              GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1749                              GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1750
1751                 /*
1752                  * Wa_1606700617:tgl
1753                  * Wa_22010271021:tgl,rkl
1754                  */
1755                 wa_masked_en(wal,
1756                              GEN9_CS_DEBUG_MODE1,
1757                              FF_DOP_CLOCK_GATE_DISABLE);
1758         }
1759
1760         if (IS_GEN(i915, 12)) {
1761                 /* Wa_1406941453:gen12 */
1762                 wa_masked_en(wal,
1763                              GEN10_SAMPLER_MODE,
1764                              ENABLE_SMALLPL);
1765         }
1766
1767         if (IS_GEN(i915, 11)) {
1768                 /* This is not an Wa. Enable for better image quality */
1769                 wa_masked_en(wal,
1770                              _3D_CHICKEN3,
1771                              _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1772
1773                 /* WaPipelineFlushCoherentLines:icl */
1774                 wa_write_or(wal,
1775                             GEN8_L3SQCREG4,
1776                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1777
1778                 /*
1779                  * Wa_1405543622:icl
1780                  * Formerly known as WaGAPZPriorityScheme
1781                  */
1782                 wa_write_or(wal,
1783                             GEN8_GARBCNTL,
1784                             GEN11_ARBITRATION_PRIO_ORDER_MASK);
1785
1786                 /*
1787                  * Wa_1604223664:icl
1788                  * Formerly known as WaL3BankAddressHashing
1789                  */
1790                 wa_write_masked_or(wal,
1791                                    GEN8_GARBCNTL,
1792                                    GEN11_HASH_CTRL_EXCL_MASK,
1793                                    GEN11_HASH_CTRL_EXCL_BIT0);
1794                 wa_write_masked_or(wal,
1795                                    GEN11_GLBLINVL,
1796                                    GEN11_BANK_HASH_ADDR_EXCL_MASK,
1797                                    GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1798
1799                 /*
1800                  * Wa_1405733216:icl
1801                  * Formerly known as WaDisableCleanEvicts
1802                  */
1803                 wa_write_or(wal,
1804                             GEN8_L3SQCREG4,
1805                             GEN11_LQSC_CLEAN_EVICT_DISABLE);
1806
1807                 /* WaForwardProgressSoftReset:icl */
1808                 wa_write_or(wal,
1809                             GEN10_SCRATCH_LNCF2,
1810                             PMFLUSHDONE_LNICRSDROP |
1811                             PMFLUSH_GAPL3UNBLOCK |
1812                             PMFLUSHDONE_LNEBLK);
1813
1814                 /* Wa_1406609255:icl (pre-prod) */
1815                 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1816                         wa_write_or(wal,
1817                                     GEN7_SARCHKMD,
1818                                     GEN7_DISABLE_DEMAND_PREFETCH);
1819
1820                 /* Wa_1606682166:icl */
1821                 wa_write_or(wal,
1822                             GEN7_SARCHKMD,
1823                             GEN7_DISABLE_SAMPLER_PREFETCH);
1824
1825                 /* Wa_1409178092:icl */
1826                 wa_write_masked_or(wal,
1827                                    GEN11_SCRATCH2,
1828                                    GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1829                                    0);
1830
1831                 /* WaEnable32PlaneMode:icl */
1832                 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
1833                              GEN11_ENABLE_32_PLANE_MODE);
1834
1835                 /*
1836                  * Wa_1408767742:icl[a2..forever],ehl[all]
1837                  * Wa_1605460711:icl[a0..c0]
1838                  */
1839                 wa_write_or(wal,
1840                             GEN7_FF_THREAD_MODE,
1841                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1842
1843                 /* Wa_22010271021:ehl */
1844                 if (IS_ELKHARTLAKE(i915))
1845                         wa_masked_en(wal,
1846                                      GEN9_CS_DEBUG_MODE1,
1847                                      FF_DOP_CLOCK_GATE_DISABLE);
1848         }
1849
1850         if (IS_GEN_RANGE(i915, 9, 12)) {
1851                 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
1852                 wa_masked_en(wal,
1853                              GEN7_FF_SLICE_CS_CHICKEN1,
1854                              GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1855         }
1856
1857         if (IS_SKYLAKE(i915) ||
1858             IS_KABYLAKE(i915) ||
1859             IS_COFFEELAKE(i915) ||
1860             IS_COMETLAKE(i915)) {
1861                 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1862                 wa_write_or(wal,
1863                             GEN8_GARBCNTL,
1864                             GEN9_GAPS_TSV_CREDIT_DISABLE);
1865         }
1866
1867         if (IS_BROXTON(i915)) {
1868                 /* WaDisablePooledEuLoadBalancingFix:bxt */
1869                 wa_masked_en(wal,
1870                              FF_SLICE_CS_CHICKEN2,
1871                              GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1872         }
1873
1874         if (IS_GEN(i915, 9)) {
1875                 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1876                 wa_masked_en(wal,
1877                              GEN9_CSFE_CHICKEN1_RCS,
1878                              GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1879
1880                 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1881                 wa_write_or(wal,
1882                             BDW_SCRATCH1,
1883                             GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1884
1885                 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1886                 if (IS_GEN9_LP(i915))
1887                         wa_write_masked_or(wal,
1888                                            GEN8_L3SQCREG1,
1889                                            L3_PRIO_CREDITS_MASK,
1890                                            L3_GENERAL_PRIO_CREDITS(62) |
1891                                            L3_HIGH_PRIO_CREDITS(2));
1892
1893                 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1894                 wa_write_or(wal,
1895                             GEN8_L3SQCREG4,
1896                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1897         }
1898
1899         if (IS_GEN(i915, 7))
1900                 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1901                 wa_masked_en(wal,
1902                              GFX_MODE_GEN7,
1903                              GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
1904
1905         if (IS_GEN_RANGE(i915, 6, 7))
1906                 /*
1907                  * We need to disable the AsyncFlip performance optimisations in
1908                  * order to use MI_WAIT_FOR_EVENT within the CS. It should
1909                  * already be programmed to '1' on all products.
1910                  *
1911                  * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1912                  */
1913                 wa_masked_en(wal,
1914                              MI_MODE,
1915                              ASYNC_FLIP_PERF_DISABLE);
1916
1917         if (IS_GEN(i915, 6)) {
1918                 /*
1919                  * Required for the hardware to program scanline values for
1920                  * waiting
1921                  * WaEnableFlushTlbInvalidationMode:snb
1922                  */
1923                 wa_masked_en(wal,
1924                              GFX_MODE,
1925                              GFX_TLB_INVALIDATE_EXPLICIT);
1926
1927                 /*
1928                  * From the Sandybridge PRM, volume 1 part 3, page 24:
1929                  * "If this bit is set, STCunit will have LRA as replacement
1930                  *  policy. [...] This bit must be reset. LRA replacement
1931                  *  policy is not supported."
1932                  */
1933                 wa_masked_dis(wal,
1934                               CACHE_MODE_0,
1935                               CM0_STC_EVICT_DISABLE_LRA_SNB);
1936         }
1937
1938         if (IS_GEN_RANGE(i915, 4, 6))
1939                 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1940                 wa_add(wal, MI_MODE,
1941                        0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
1942                        /* XXX bit doesn't stick on Broadwater */
1943                        IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH);
1944
1945         if (IS_GEN(i915, 4))
1946                 /*
1947                  * Disable CONSTANT_BUFFER before it is loaded from the context
1948                  * image. For as it is loaded, it is executed and the stored
1949                  * address may no longer be valid, leading to a GPU hang.
1950                  *
1951                  * This imposes the requirement that userspace reload their
1952                  * CONSTANT_BUFFER on every batch, fortunately a requirement
1953                  * they are already accustomed to from before contexts were
1954                  * enabled.
1955                  */
1956                 wa_add(wal, ECOSKPD,
1957                        0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
1958                        0 /* XXX bit doesn't stick on Broadwater */);
1959 }
1960
1961 static void
1962 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1963 {
1964         struct drm_i915_private *i915 = engine->i915;
1965
1966         /* WaKBLVECSSemaphoreWaitPoll:kbl */
1967         if (IS_KBL_GT_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
1968                 wa_write(wal,
1969                          RING_SEMA_WAIT_POLL(engine->mmio_base),
1970                          1);
1971         }
1972 }
1973
1974 static void
1975 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1976 {
1977         if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 4))
1978                 return;
1979
1980         if (engine->class == RENDER_CLASS)
1981                 rcs_engine_wa_init(engine, wal);
1982         else
1983                 xcs_engine_wa_init(engine, wal);
1984 }
1985
1986 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
1987 {
1988         struct i915_wa_list *wal = &engine->wa_list;
1989
1990         if (INTEL_GEN(engine->i915) < 4)
1991                 return;
1992
1993         wa_init_start(wal, "engine", engine->name);
1994         engine_init_workarounds(engine, wal);
1995         wa_init_finish(wal);
1996 }
1997
1998 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
1999 {
2000         wa_list_apply(engine->uncore, &engine->wa_list);
2001 }
2002
2003 static struct i915_vma *
2004 create_scratch(struct i915_address_space *vm, int count)
2005 {
2006         struct drm_i915_gem_object *obj;
2007         struct i915_vma *vma;
2008         unsigned int size;
2009         int err;
2010
2011         size = round_up(count * sizeof(u32), PAGE_SIZE);
2012         obj = i915_gem_object_create_internal(vm->i915, size);
2013         if (IS_ERR(obj))
2014                 return ERR_CAST(obj);
2015
2016         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
2017
2018         vma = i915_vma_instance(obj, vm, NULL);
2019         if (IS_ERR(vma)) {
2020                 err = PTR_ERR(vma);
2021                 goto err_obj;
2022         }
2023
2024         err = i915_vma_pin(vma, 0, 0,
2025                            i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
2026         if (err)
2027                 goto err_obj;
2028
2029         return vma;
2030
2031 err_obj:
2032         i915_gem_object_put(obj);
2033         return ERR_PTR(err);
2034 }
2035
2036 static const struct {
2037         u32 start;
2038         u32 end;
2039 } mcr_ranges_gen8[] = {
2040         { .start = 0x5500, .end = 0x55ff },
2041         { .start = 0x7000, .end = 0x7fff },
2042         { .start = 0x9400, .end = 0x97ff },
2043         { .start = 0xb000, .end = 0xb3ff },
2044         { .start = 0xe000, .end = 0xe7ff },
2045         {},
2046 };
2047
2048 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
2049 {
2050         int i;
2051
2052         if (INTEL_GEN(i915) < 8)
2053                 return false;
2054
2055         /*
2056          * Registers in these ranges are affected by the MCR selector
2057          * which only controls CPU initiated MMIO. Routing does not
2058          * work for CS access so we cannot verify them on this path.
2059          */
2060         for (i = 0; mcr_ranges_gen8[i].start; i++)
2061                 if (offset >= mcr_ranges_gen8[i].start &&
2062                     offset <= mcr_ranges_gen8[i].end)
2063                         return true;
2064
2065         return false;
2066 }
2067
2068 static int
2069 wa_list_srm(struct i915_request *rq,
2070             const struct i915_wa_list *wal,
2071             struct i915_vma *vma)
2072 {
2073         struct drm_i915_private *i915 = rq->engine->i915;
2074         unsigned int i, count = 0;
2075         const struct i915_wa *wa;
2076         u32 srm, *cs;
2077
2078         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
2079         if (INTEL_GEN(i915) >= 8)
2080                 srm++;
2081
2082         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2083                 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
2084                         count++;
2085         }
2086
2087         cs = intel_ring_begin(rq, 4 * count);
2088         if (IS_ERR(cs))
2089                 return PTR_ERR(cs);
2090
2091         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2092                 u32 offset = i915_mmio_reg_offset(wa->reg);
2093
2094                 if (mcr_range(i915, offset))
2095                         continue;
2096
2097                 *cs++ = srm;
2098                 *cs++ = offset;
2099                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
2100                 *cs++ = 0;
2101         }
2102         intel_ring_advance(rq, cs);
2103
2104         return 0;
2105 }
2106
2107 static int engine_wa_list_verify(struct intel_context *ce,
2108                                  const struct i915_wa_list * const wal,
2109                                  const char *from)
2110 {
2111         const struct i915_wa *wa;
2112         struct i915_request *rq;
2113         struct i915_vma *vma;
2114         struct i915_gem_ww_ctx ww;
2115         unsigned int i;
2116         u32 *results;
2117         int err;
2118
2119         if (!wal->count)
2120                 return 0;
2121
2122         vma = create_scratch(&ce->engine->gt->ggtt->vm, wal->count);
2123         if (IS_ERR(vma))
2124                 return PTR_ERR(vma);
2125
2126         intel_engine_pm_get(ce->engine);
2127         i915_gem_ww_ctx_init(&ww, false);
2128 retry:
2129         err = i915_gem_object_lock(vma->obj, &ww);
2130         if (err == 0)
2131                 err = intel_context_pin_ww(ce, &ww);
2132         if (err)
2133                 goto err_pm;
2134
2135         rq = i915_request_create(ce);
2136         if (IS_ERR(rq)) {
2137                 err = PTR_ERR(rq);
2138                 goto err_unpin;
2139         }
2140
2141         err = i915_request_await_object(rq, vma->obj, true);
2142         if (err == 0)
2143                 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
2144         if (err == 0)
2145                 err = wa_list_srm(rq, wal, vma);
2146
2147         i915_request_get(rq);
2148         if (err)
2149                 i915_request_set_error_once(rq, err);
2150         i915_request_add(rq);
2151
2152         if (err)
2153                 goto err_rq;
2154
2155         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2156                 err = -ETIME;
2157                 goto err_rq;
2158         }
2159
2160         results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
2161         if (IS_ERR(results)) {
2162                 err = PTR_ERR(results);
2163                 goto err_rq;
2164         }
2165
2166         err = 0;
2167         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2168                 if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
2169                         continue;
2170
2171                 if (!wa_verify(wa, results[i], wal->name, from))
2172                         err = -ENXIO;
2173         }
2174
2175         i915_gem_object_unpin_map(vma->obj);
2176
2177 err_rq:
2178         i915_request_put(rq);
2179 err_unpin:
2180         intel_context_unpin(ce);
2181 err_pm:
2182         if (err == -EDEADLK) {
2183                 err = i915_gem_ww_ctx_backoff(&ww);
2184                 if (!err)
2185                         goto retry;
2186         }
2187         i915_gem_ww_ctx_fini(&ww);
2188         intel_engine_pm_put(ce->engine);
2189         i915_vma_unpin(vma);
2190         i915_vma_put(vma);
2191         return err;
2192 }
2193
2194 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
2195                                     const char *from)
2196 {
2197         return engine_wa_list_verify(engine->kernel_context,
2198                                      &engine->wa_list,
2199                                      from);
2200 }
2201
2202 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2203 #include "selftest_workarounds.c"
2204 #endif