drivers/gpu/drm/i915/gt/selftest_workarounds.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2018 Intel Corporation
   5  */
   6
   7 #include "gem/i915_gem_pm.h"
   8 #include "gt/intel_engine_user.h"
   9 #include "gt/intel_gt.h"
  10 #include "i915_selftest.h"
  11 #include "intel_reset.h"
  12
  13 #include "selftests/igt_flush_test.h"
  14 #include "selftests/igt_reset.h"
  15 #include "selftests/igt_spinner.h"
  16 #include "selftests/mock_drm.h"
  17
  18 #include "gem/selftests/igt_gem_utils.h"
  19 #include "gem/selftests/mock_context.h"
  20
  21 static const struct wo_register {
  22         enum intel_platform platform;
  23         u32 reg;
  24 } wo_registers[] = {
  25         { INTEL_GEMINILAKE, 0x731c }
  26 };
  27
  28 struct wa_lists {
  29         struct i915_wa_list gt_wa_list;
  30         struct {
  31                 struct i915_wa_list wa_list;
  32                 struct i915_wa_list ctx_wa_list;
  33         } engine[I915_NUM_ENGINES];
  34 };
  35
  36 static int request_add_sync(struct i915_request *rq, int err)
  37 {
  38         i915_request_get(rq);
  39         i915_request_add(rq);
  40         if (i915_request_wait(rq, 0, HZ / 5) < 0)
  41                 err = -EIO;
  42         i915_request_put(rq);
  43
  44         return err;
  45 }
  46
  47 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)
  48 {
  49         int err = 0;
  50
  51         i915_request_get(rq);
  52         i915_request_add(rq);
  53         if (spin && !igt_wait_for_spinner(spin, rq))
  54                 err = -ETIMEDOUT;
  55         i915_request_put(rq);
  56
  57         return err;
  58 }
  59
  60 static void
  61 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)
  62 {
  63         struct intel_engine_cs *engine;
  64         enum intel_engine_id id;
  65
  66         memset(lists, 0, sizeof(*lists));
  67
  68         wa_init_start(&lists->gt_wa_list, "GT_REF", "global");
  69         gt_init_workarounds(gt->i915, &lists->gt_wa_list);
  70         wa_init_finish(&lists->gt_wa_list);
  71
  72         for_each_engine(engine, gt, id) {
  73                 struct i915_wa_list *wal = &lists->engine[id].wa_list;
  74
  75                 wa_init_start(wal, "REF", engine->name);
  76                 engine_init_workarounds(engine, wal);
  77                 wa_init_finish(wal);
  78
  79                 __intel_engine_init_ctx_wa(engine,
  80                                            &lists->engine[id].ctx_wa_list,
  81                                            "CTX_REF");
  82         }
  83 }
  84
  85 static void
  86 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)
  87 {
  88         struct intel_engine_cs *engine;
  89         enum intel_engine_id id;
  90
  91         for_each_engine(engine, gt, id)
  92                 intel_wa_list_free(&lists->engine[id].wa_list);
  93
  94         intel_wa_list_free(&lists->gt_wa_list);
  95 }
  96
  97 static struct drm_i915_gem_object *
  98 read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
  99 {
 100         const u32 base = engine->mmio_base;
 101         struct drm_i915_gem_object *result;
 102         struct i915_request *rq;
 103         struct i915_vma *vma;
 104         u32 srm, *cs;
 105         int err;
 106         int i;
 107
 108         result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
 109         if (IS_ERR(result))
 110                 return result;
 111
 112         i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
 113
 114         cs = i915_gem_object_pin_map(result, I915_MAP_WB);
 115         if (IS_ERR(cs)) {
 116                 err = PTR_ERR(cs);
 117                 goto err_obj;
 118         }
 119         memset(cs, 0xc5, PAGE_SIZE);
 120         i915_gem_object_flush_map(result);
 121         i915_gem_object_unpin_map(result);
 122
 123         vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);
 124         if (IS_ERR(vma)) {
 125                 err = PTR_ERR(vma);
 126                 goto err_obj;
 127         }
 128
 129         err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
 130         if (err)
 131                 goto err_obj;
 132
 133         rq = igt_request_alloc(ctx, engine);
 134         if (IS_ERR(rq)) {
 135                 err = PTR_ERR(rq);
 136                 goto err_pin;
 137         }
 138
 139         i915_vma_lock(vma);
 140         err = i915_request_await_object(rq, vma->obj, true);
 141         if (err == 0)
 142                 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
 143         i915_vma_unlock(vma);
 144         if (err)
 145                 goto err_req;
 146
 147         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
 148         if (INTEL_GEN(ctx->i915) >= 8)
 149                 srm++;
 150
 151         cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
 152         if (IS_ERR(cs)) {
 153                 err = PTR_ERR(cs);
 154                 goto err_req;
 155         }
 156
 157         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 158                 *cs++ = srm;
 159                 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
 160                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
 161                 *cs++ = 0;
 162         }
 163         intel_ring_advance(rq, cs);
 164
 165         i915_request_add(rq);
 166         i915_vma_unpin(vma);
 167
 168         return result;
 169
 170 err_req:
 171         i915_request_add(rq);
 172 err_pin:
 173         i915_vma_unpin(vma);
 174 err_obj:
 175         i915_gem_object_put(result);
 176         return ERR_PTR(err);
 177 }
 178
 179 static u32
 180 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)
 181 {
 182         i915_reg_t reg = i < engine->whitelist.count ?
 183                          engine->whitelist.list[i].reg :
 184                          RING_NOPID(engine->mmio_base);
 185
 186         return i915_mmio_reg_offset(reg);
 187 }
 188
 189 static void
 190 print_results(const struct intel_engine_cs *engine, const u32 *results)
 191 {
 192         unsigned int i;
 193
 194         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 195                 u32 expected = get_whitelist_reg(engine, i);
 196                 u32 actual = results[i];
 197
 198                 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
 199                         i, expected, actual);
 200         }
 201 }
 202
 203 static int check_whitelist(struct i915_gem_context *ctx,
 204                            struct intel_engine_cs *engine)
 205 {
 206         struct drm_i915_gem_object *results;
 207         struct intel_wedge_me wedge;
 208         u32 *vaddr;
 209         int err;
 210         int i;
 211
 212         results = read_nonprivs(ctx, engine);
 213         if (IS_ERR(results))
 214                 return PTR_ERR(results);
 215
 216         err = 0;
 217         i915_gem_object_lock(results, NULL);
 218         intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */
 219                 err = i915_gem_object_set_to_cpu_domain(results, false);
 220         i915_gem_object_unlock(results);
 221         if (intel_gt_is_wedged(engine->gt))
 222                 err = -EIO;
 223         if (err)
 224                 goto out_put;
 225
 226         vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
 227         if (IS_ERR(vaddr)) {
 228                 err = PTR_ERR(vaddr);
 229                 goto out_put;
 230         }
 231
 232         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 233                 u32 expected = get_whitelist_reg(engine, i);
 234                 u32 actual = vaddr[i];
 235
 236                 if (expected != actual) {
 237                         print_results(engine, vaddr);
 238                         pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
 239                                i, expected, actual);
 240
 241                         err = -EINVAL;
 242                         break;
 243                 }
 244         }
 245
 246         i915_gem_object_unpin_map(results);
 247 out_put:
 248         i915_gem_object_put(results);
 249         return err;
 250 }
 251
 252 static int do_device_reset(struct intel_engine_cs *engine)
 253 {
 254         intel_gt_reset(engine->gt, engine->mask, "live_workarounds");
 255         return 0;
 256 }
 257
 258 static int do_engine_reset(struct intel_engine_cs *engine)
 259 {
 260         return intel_engine_reset(engine, "live_workarounds");
 261 }
 262
 263 static int
 264 switch_to_scratch_context(struct intel_engine_cs *engine,
 265                           struct igt_spinner *spin)
 266 {
 267         struct intel_context *ce;
 268         struct i915_request *rq;
 269         int err = 0;
 270
 271         ce = intel_context_create(engine);
 272         if (IS_ERR(ce))
 273                 return PTR_ERR(ce);
 274
 275         rq = igt_spinner_create_request(spin, ce, MI_NOOP);
 276         intel_context_put(ce);
 277
 278         if (IS_ERR(rq)) {
 279                 spin = NULL;
 280                 err = PTR_ERR(rq);
 281                 goto err;
 282         }
 283
 284         err = request_add_spin(rq, spin);
 285 err:
 286         if (err && spin)
 287                 igt_spinner_end(spin);
 288
 289         return err;
 290 }
 291
 292 static int check_whitelist_across_reset(struct intel_engine_cs *engine,
 293                                         int (*reset)(struct intel_engine_cs *),
 294                                         const char *name)
 295 {
 296         struct drm_i915_private *i915 = engine->i915;
 297         struct i915_gem_context *ctx, *tmp;
 298         struct igt_spinner spin;
 299         intel_wakeref_t wakeref;
 300         int err;
 301
 302         pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",
 303                 engine->whitelist.count, engine->name, name);
 304
 305         ctx = kernel_context(i915);
 306         if (IS_ERR(ctx))
 307                 return PTR_ERR(ctx);
 308
 309         err = igt_spinner_init(&spin, engine->gt);
 310         if (err)
 311                 goto out_ctx;
 312
 313         err = check_whitelist(ctx, engine);
 314         if (err) {
 315                 pr_err("Invalid whitelist *before* %s reset!\n", name);
 316                 goto out_spin;
 317         }
 318
 319         err = switch_to_scratch_context(engine, &spin);
 320         if (err)
 321                 goto out_spin;
 322
 323         with_intel_runtime_pm(engine->uncore->rpm, wakeref)
 324                 err = reset(engine);
 325
 326         igt_spinner_end(&spin);
 327
 328         if (err) {
 329                 pr_err("%s reset failed\n", name);
 330                 goto out_spin;
 331         }
 332
 333         err = check_whitelist(ctx, engine);
 334         if (err) {
 335                 pr_err("Whitelist not preserved in context across %s reset!\n",
 336                        name);
 337                 goto out_spin;
 338         }
 339
 340         tmp = kernel_context(i915);
 341         if (IS_ERR(tmp)) {
 342                 err = PTR_ERR(tmp);
 343                 goto out_spin;
 344         }
 345         kernel_context_close(ctx);
 346         ctx = tmp;
 347
 348         err = check_whitelist(ctx, engine);
 349         if (err) {
 350                 pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
 351                        name);
 352                 goto out_spin;
 353         }
 354
 355 out_spin:
 356         igt_spinner_fini(&spin);
 357 out_ctx:
 358         kernel_context_close(ctx);
 359         return err;
 360 }
 361
 362 static struct i915_vma *create_batch(struct i915_address_space *vm)
 363 {
 364         struct drm_i915_gem_object *obj;
 365         struct i915_vma *vma;
 366         int err;
 367
 368         obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);
 369         if (IS_ERR(obj))
 370                 return ERR_CAST(obj);
 371
 372         vma = i915_vma_instance(obj, vm, NULL);
 373         if (IS_ERR(vma)) {
 374                 err = PTR_ERR(vma);
 375                 goto err_obj;
 376         }
 377
 378         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 379         if (err)
 380                 goto err_obj;
 381
 382         return vma;
 383
 384 err_obj:
 385         i915_gem_object_put(obj);
 386         return ERR_PTR(err);
 387 }
 388
 389 static u32 reg_write(u32 old, u32 new, u32 rsvd)
 390 {
 391         if (rsvd == 0x0000ffff) {
 392                 old &= ~(new >> 16);
 393                 old |= new & (new >> 16);
 394         } else {
 395                 old &= ~rsvd;
 396                 old |= new & rsvd;
 397         }
 398
 399         return old;
 400 }
 401
 402 static bool wo_register(struct intel_engine_cs *engine, u32 reg)
 403 {
 404         enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
 405         int i;
 406
 407         if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
 408              RING_FORCE_TO_NONPRIV_ACCESS_WR)
 409                 return true;
 410
 411         for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
 412                 if (wo_registers[i].platform == platform &&
 413                     wo_registers[i].reg == reg)
 414                         return true;
 415         }
 416
 417         return false;
 418 }
 419
 420 static bool timestamp(const struct intel_engine_cs *engine, u32 reg)
 421 {
 422         reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;
 423         switch (reg) {
 424         case 0x358:
 425         case 0x35c:
 426         case 0x3a8:
 427                 return true;
 428
 429         default:
 430                 return false;
 431         }
 432 }
 433
 434 static bool ro_register(u32 reg)
 435 {
 436         if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
 437              RING_FORCE_TO_NONPRIV_ACCESS_RD)
 438                 return true;
 439
 440         return false;
 441 }
 442
 443 static int whitelist_writable_count(struct intel_engine_cs *engine)
 444 {
 445         int count = engine->whitelist.count;
 446         int i;
 447
 448         for (i = 0; i < engine->whitelist.count; i++) {
 449                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 450
 451                 if (ro_register(reg))
 452                         count--;
 453         }
 454
 455         return count;
 456 }
 457
 458 static int check_dirty_whitelist(struct intel_context *ce)
 459 {
 460         const u32 values[] = {
 461                 0x00000000,
 462                 0x01010101,
 463                 0x10100101,
 464                 0x03030303,
 465                 0x30300303,
 466                 0x05050505,
 467                 0x50500505,
 468                 0x0f0f0f0f,
 469                 0xf00ff00f,
 470                 0x10101010,
 471                 0xf0f01010,
 472                 0x30303030,
 473                 0xa0a03030,
 474                 0x50505050,
 475                 0xc0c05050,
 476                 0xf0f0f0f0,
 477                 0x11111111,
 478                 0x33333333,
 479                 0x55555555,
 480                 0x0000ffff,
 481                 0x00ff00ff,
 482                 0xff0000ff,
 483                 0xffff00ff,
 484                 0xffffffff,
 485         };
 486         struct intel_engine_cs *engine = ce->engine;
 487         struct i915_vma *scratch;
 488         struct i915_vma *batch;
 489         int err = 0, i, v;
 490         u32 *cs, *results;
 491
 492         scratch = create_scratch(ce->vm, 2 * ARRAY_SIZE(values) + 1);
 493         if (IS_ERR(scratch))
 494                 return PTR_ERR(scratch);
 495
 496         batch = create_batch(ce->vm);
 497         if (IS_ERR(batch)) {
 498                 err = PTR_ERR(batch);
 499                 goto out_scratch;
 500         }
 501
 502         for (i = 0; i < engine->whitelist.count; i++) {
 503                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 504                 u64 addr = scratch->node.start;
 505                 struct i915_request *rq;
 506                 u32 srm, lrm, rsvd;
 507                 u32 expect;
 508                 int idx;
 509                 bool ro_reg;
 510
 511                 if (wo_register(engine, reg))
 512                         continue;
 513
 514                 if (timestamp(engine, reg))
 515                         continue; /* timestamps are expected to autoincrement */
 516
 517                 ro_reg = ro_register(reg);
 518
 519                 /* Clear non priv flags */
 520                 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 521
 522                 srm = MI_STORE_REGISTER_MEM;
 523                 lrm = MI_LOAD_REGISTER_MEM;
 524                 if (INTEL_GEN(engine->i915) >= 8)
 525                         lrm++, srm++;
 526
 527                 pr_debug("%s: Writing garbage to %x\n",
 528                          engine->name, reg);
 529
 530                 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
 531                 if (IS_ERR(cs)) {
 532                         err = PTR_ERR(cs);
 533                         goto out_batch;
 534                 }
 535
 536                 /* SRM original */
 537                 *cs++ = srm;
 538                 *cs++ = reg;
 539                 *cs++ = lower_32_bits(addr);
 540                 *cs++ = upper_32_bits(addr);
 541
 542                 idx = 1;
 543                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 544                         /* LRI garbage */
 545                         *cs++ = MI_LOAD_REGISTER_IMM(1);
 546                         *cs++ = reg;
 547                         *cs++ = values[v];
 548
 549                         /* SRM result */
 550                         *cs++ = srm;
 551                         *cs++ = reg;
 552                         *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
 553                         *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
 554                         idx++;
 555                 }
 556                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 557                         /* LRI garbage */
 558                         *cs++ = MI_LOAD_REGISTER_IMM(1);
 559                         *cs++ = reg;
 560                         *cs++ = ~values[v];
 561
 562                         /* SRM result */
 563                         *cs++ = srm;
 564                         *cs++ = reg;
 565                         *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
 566                         *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
 567                         idx++;
 568                 }
 569                 GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
 570
 571                 /* LRM original -- don't leave garbage in the context! */
 572                 *cs++ = lrm;
 573                 *cs++ = reg;
 574                 *cs++ = lower_32_bits(addr);
 575                 *cs++ = upper_32_bits(addr);
 576
 577                 *cs++ = MI_BATCH_BUFFER_END;
 578
 579                 i915_gem_object_flush_map(batch->obj);
 580                 i915_gem_object_unpin_map(batch->obj);
 581                 intel_gt_chipset_flush(engine->gt);
 582
 583                 rq = intel_context_create_request(ce);
 584                 if (IS_ERR(rq)) {
 585                         err = PTR_ERR(rq);
 586                         goto out_batch;
 587                 }
 588
 589                 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
 590                         err = engine->emit_init_breadcrumb(rq);
 591                         if (err)
 592                                 goto err_request;
 593                 }
 594
 595                 i915_vma_lock(batch);
 596                 err = i915_request_await_object(rq, batch->obj, false);
 597                 if (err == 0)
 598                         err = i915_vma_move_to_active(batch, rq, 0);
 599                 i915_vma_unlock(batch);
 600                 if (err)
 601                         goto err_request;
 602
 603                 i915_vma_lock(scratch);
 604                 err = i915_request_await_object(rq, scratch->obj, true);
 605                 if (err == 0)
 606                         err = i915_vma_move_to_active(scratch, rq,
 607                                                       EXEC_OBJECT_WRITE);
 608                 i915_vma_unlock(scratch);
 609                 if (err)
 610                         goto err_request;
 611
 612                 err = engine->emit_bb_start(rq,
 613                                             batch->node.start, PAGE_SIZE,
 614                                             0);
 615                 if (err)
 616                         goto err_request;
 617
 618 err_request:
 619                 err = request_add_sync(rq, err);
 620                 if (err) {
 621                         pr_err("%s: Futzing %x timedout; cancelling test\n",
 622                                engine->name, reg);
 623                         intel_gt_set_wedged(engine->gt);
 624                         goto out_batch;
 625                 }
 626
 627                 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
 628                 if (IS_ERR(results)) {
 629                         err = PTR_ERR(results);
 630                         goto out_batch;
 631                 }
 632
 633                 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
 634                 if (!ro_reg) {
 635                         /* detect write masking */
 636                         rsvd = results[ARRAY_SIZE(values)];
 637                         if (!rsvd) {
 638                                 pr_err("%s: Unable to write to whitelisted register %x\n",
 639                                        engine->name, reg);
 640                                 err = -EINVAL;
 641                                 goto out_unpin;
 642                         }
 643                 } else {
 644                         rsvd = 0;
 645                 }
 646
 647                 expect = results[0];
 648                 idx = 1;
 649                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 650                         if (ro_reg)
 651                                 expect = results[0];
 652                         else
 653                                 expect = reg_write(expect, values[v], rsvd);
 654
 655                         if (results[idx] != expect)
 656                                 err++;
 657                         idx++;
 658                 }
 659                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 660                         if (ro_reg)
 661                                 expect = results[0];
 662                         else
 663                                 expect = reg_write(expect, ~values[v], rsvd);
 664
 665                         if (results[idx] != expect)
 666                                 err++;
 667                         idx++;
 668                 }
 669                 if (err) {
 670                         pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
 671                                engine->name, err, reg);
 672
 673                         if (ro_reg)
 674                                 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",
 675                                         engine->name, reg, results[0]);
 676                         else
 677                                 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
 678                                         engine->name, reg, results[0], rsvd);
 679
 680                         expect = results[0];
 681                         idx = 1;
 682                         for (v = 0; v < ARRAY_SIZE(values); v++) {
 683                                 u32 w = values[v];
 684
 685                                 if (ro_reg)
 686                                         expect = results[0];
 687                                 else
 688                                         expect = reg_write(expect, w, rsvd);
 689                                 pr_info("Wrote %08x, read %08x, expect %08x\n",
 690                                         w, results[idx], expect);
 691                                 idx++;
 692                         }
 693                         for (v = 0; v < ARRAY_SIZE(values); v++) {
 694                                 u32 w = ~values[v];
 695
 696                                 if (ro_reg)
 697                                         expect = results[0];
 698                                 else
 699                                         expect = reg_write(expect, w, rsvd);
 700                                 pr_info("Wrote %08x, read %08x, expect %08x\n",
 701                                         w, results[idx], expect);
 702                                 idx++;
 703                         }
 704
 705                         err = -EINVAL;
 706                 }
 707 out_unpin:
 708                 i915_gem_object_unpin_map(scratch->obj);
 709                 if (err)
 710                         break;
 711         }
 712
 713         if (igt_flush_test(engine->i915))
 714                 err = -EIO;
 715 out_batch:
 716         i915_vma_unpin_and_release(&batch, 0);
 717 out_scratch:
 718         i915_vma_unpin_and_release(&scratch, 0);
 719         return err;
 720 }
 721
 722 static int live_dirty_whitelist(void *arg)
 723 {
 724         struct intel_gt *gt = arg;
 725         struct intel_engine_cs *engine;
 726         enum intel_engine_id id;
 727
 728         /* Can the user write to the whitelisted registers? */
 729
 730         if (INTEL_GEN(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */
 731                 return 0;
 732
 733         for_each_engine(engine, gt, id) {
 734                 struct intel_context *ce;
 735                 int err;
 736
 737                 if (engine->whitelist.count == 0)
 738                         continue;
 739
 740                 ce = intel_context_create(engine);
 741                 if (IS_ERR(ce))
 742                         return PTR_ERR(ce);
 743
 744                 err = check_dirty_whitelist(ce);
 745                 intel_context_put(ce);
 746                 if (err)
 747                         return err;
 748         }
 749
 750         return 0;
 751 }
 752
 753 static int live_reset_whitelist(void *arg)
 754 {
 755         struct intel_gt *gt = arg;
 756         struct intel_engine_cs *engine;
 757         enum intel_engine_id id;
 758         int err = 0;
 759
 760         /* If we reset the gpu, we should not lose the RING_NONPRIV */
 761         igt_global_reset_lock(gt);
 762
 763         for_each_engine(engine, gt, id) {
 764                 if (engine->whitelist.count == 0)
 765                         continue;
 766
 767                 if (intel_has_reset_engine(gt)) {
 768                         err = check_whitelist_across_reset(engine,
 769                                                            do_engine_reset,
 770                                                            "engine");
 771                         if (err)
 772                                 goto out;
 773                 }
 774
 775                 if (intel_has_gpu_reset(gt)) {
 776                         err = check_whitelist_across_reset(engine,
 777                                                            do_device_reset,
 778                                                            "device");
 779                         if (err)
 780                                 goto out;
 781                 }
 782         }
 783
 784 out:
 785         igt_global_reset_unlock(gt);
 786         return err;
 787 }
 788
 789 static int read_whitelisted_registers(struct i915_gem_context *ctx,
 790                                       struct intel_engine_cs *engine,
 791                                       struct i915_vma *results)
 792 {
 793         struct i915_request *rq;
 794         int i, err = 0;
 795         u32 srm, *cs;
 796
 797         rq = igt_request_alloc(ctx, engine);
 798         if (IS_ERR(rq))
 799                 return PTR_ERR(rq);
 800
 801         i915_vma_lock(results);
 802         err = i915_request_await_object(rq, results->obj, true);
 803         if (err == 0)
 804                 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE);
 805         i915_vma_unlock(results);
 806         if (err)
 807                 goto err_req;
 808
 809         srm = MI_STORE_REGISTER_MEM;
 810         if (INTEL_GEN(ctx->i915) >= 8)
 811                 srm++;
 812
 813         cs = intel_ring_begin(rq, 4 * engine->whitelist.count);
 814         if (IS_ERR(cs)) {
 815                 err = PTR_ERR(cs);
 816                 goto err_req;
 817         }
 818
 819         for (i = 0; i < engine->whitelist.count; i++) {
 820                 u64 offset = results->node.start + sizeof(u32) * i;
 821                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 822
 823                 /* Clear non priv flags */
 824                 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 825
 826                 *cs++ = srm;
 827                 *cs++ = reg;
 828                 *cs++ = lower_32_bits(offset);
 829                 *cs++ = upper_32_bits(offset);
 830         }
 831         intel_ring_advance(rq, cs);
 832
 833 err_req:
 834         return request_add_sync(rq, err);
 835 }
 836
 837 static int scrub_whitelisted_registers(struct i915_gem_context *ctx,
 838                                        struct intel_engine_cs *engine)
 839 {
 840         struct i915_address_space *vm;
 841         struct i915_request *rq;
 842         struct i915_vma *batch;
 843         int i, err = 0;
 844         u32 *cs;
 845
 846         vm = i915_gem_context_get_vm_rcu(ctx);
 847         batch = create_batch(vm);
 848         i915_vm_put(vm);
 849         if (IS_ERR(batch))
 850                 return PTR_ERR(batch);
 851
 852         cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
 853         if (IS_ERR(cs)) {
 854                 err = PTR_ERR(cs);
 855                 goto err_batch;
 856         }
 857
 858         *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));
 859         for (i = 0; i < engine->whitelist.count; i++) {
 860                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 861
 862                 if (ro_register(reg))
 863                         continue;
 864
 865                 /* Clear non priv flags */
 866                 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 867
 868                 *cs++ = reg;
 869                 *cs++ = 0xffffffff;
 870         }
 871         *cs++ = MI_BATCH_BUFFER_END;
 872
 873         i915_gem_object_flush_map(batch->obj);
 874         intel_gt_chipset_flush(engine->gt);
 875
 876         rq = igt_request_alloc(ctx, engine);
 877         if (IS_ERR(rq)) {
 878                 err = PTR_ERR(rq);
 879                 goto err_unpin;
 880         }
 881
 882         if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
 883                 err = engine->emit_init_breadcrumb(rq);
 884                 if (err)
 885                         goto err_request;
 886         }
 887
 888         i915_vma_lock(batch);
 889         err = i915_request_await_object(rq, batch->obj, false);
 890         if (err == 0)
 891                 err = i915_vma_move_to_active(batch, rq, 0);
 892         i915_vma_unlock(batch);
 893         if (err)
 894                 goto err_request;
 895
 896         /* Perform the writes from an unprivileged "user" batch */
 897         err = engine->emit_bb_start(rq, batch->node.start, 0, 0);
 898
 899 err_request:
 900         err = request_add_sync(rq, err);
 901
 902 err_unpin:
 903         i915_gem_object_unpin_map(batch->obj);
 904 err_batch:
 905         i915_vma_unpin_and_release(&batch, 0);
 906         return err;
 907 }
 908
 909 struct regmask {
 910         i915_reg_t reg;
 911         unsigned long gen_mask;
 912 };
 913
 914 static bool find_reg(struct drm_i915_private *i915,
 915                      i915_reg_t reg,
 916                      const struct regmask *tbl,
 917                      unsigned long count)
 918 {
 919         u32 offset = i915_mmio_reg_offset(reg);
 920
 921         while (count--) {
 922                 if (INTEL_INFO(i915)->gen_mask & tbl->gen_mask &&
 923                     i915_mmio_reg_offset(tbl->reg) == offset)
 924                         return true;
 925                 tbl++;
 926         }
 927
 928         return false;
 929 }
 930
 931 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)
 932 {
 933         /* Alas, we must pardon some whitelists. Mistakes already made */
 934         static const struct regmask pardon[] = {
 935                 { GEN9_CTX_PREEMPT_REG, INTEL_GEN_MASK(9, 9) },
 936                 { GEN8_L3SQCREG4, INTEL_GEN_MASK(9, 9) },
 937         };
 938
 939         return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));
 940 }
 941
 942 static bool result_eq(struct intel_engine_cs *engine,
 943                       u32 a, u32 b, i915_reg_t reg)
 944 {
 945         if (a != b && !pardon_reg(engine->i915, reg)) {
 946                 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",
 947                        i915_mmio_reg_offset(reg), a, b);
 948                 return false;
 949         }
 950
 951         return true;
 952 }
 953
 954 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)
 955 {
 956         /* Some registers do not seem to behave and our writes unreadable */
 957         static const struct regmask wo[] = {
 958                 { GEN9_SLICE_COMMON_ECO_CHICKEN1, INTEL_GEN_MASK(9, 9) },
 959         };
 960
 961         return find_reg(i915, reg, wo, ARRAY_SIZE(wo));
 962 }
 963
 964 static bool result_neq(struct intel_engine_cs *engine,
 965                        u32 a, u32 b, i915_reg_t reg)
 966 {
 967         if (a == b && !writeonly_reg(engine->i915, reg)) {
 968                 pr_err("Whitelist register 0x%4x:%08x was unwritable\n",
 969                        i915_mmio_reg_offset(reg), a);
 970                 return false;
 971         }
 972
 973         return true;
 974 }
 975
 976 static int
 977 check_whitelisted_registers(struct intel_engine_cs *engine,
 978                             struct i915_vma *A,
 979                             struct i915_vma *B,
 980                             bool (*fn)(struct intel_engine_cs *engine,
 981                                        u32 a, u32 b,
 982                                        i915_reg_t reg))
 983 {
 984         u32 *a, *b;
 985         int i, err;
 986
 987         a = i915_gem_object_pin_map(A->obj, I915_MAP_WB);
 988         if (IS_ERR(a))
 989                 return PTR_ERR(a);
 990
 991         b = i915_gem_object_pin_map(B->obj, I915_MAP_WB);
 992         if (IS_ERR(b)) {
 993                 err = PTR_ERR(b);
 994                 goto err_a;
 995         }
 996
 997         err = 0;
 998         for (i = 0; i < engine->whitelist.count; i++) {
 999                 const struct i915_wa *wa = &engine->whitelist.list[i];
1000
1001                 if (i915_mmio_reg_offset(wa->reg) &
1002                     RING_FORCE_TO_NONPRIV_ACCESS_RD)
1003                         continue;
1004
1005                 if (!fn(engine, a[i], b[i], wa->reg))
1006                         err = -EINVAL;
1007         }
1008
1009         i915_gem_object_unpin_map(B->obj);
1010 err_a:
1011         i915_gem_object_unpin_map(A->obj);
1012         return err;
1013 }
1014
1015 static int live_isolated_whitelist(void *arg)
1016 {
1017         struct intel_gt *gt = arg;
1018         struct {
1019                 struct i915_gem_context *ctx;
1020                 struct i915_vma *scratch[2];
1021         } client[2] = {};
1022         struct intel_engine_cs *engine;
1023         enum intel_engine_id id;
1024         int i, err = 0;
1025
1026         /*
1027          * Check that a write into a whitelist register works, but
1028          * invisible to a second context.
1029          */
1030
1031         if (!intel_engines_has_context_isolation(gt->i915))
1032                 return 0;
1033
1034         for (i = 0; i < ARRAY_SIZE(client); i++) {
1035                 struct i915_address_space *vm;
1036                 struct i915_gem_context *c;
1037
1038                 c = kernel_context(gt->i915);
1039                 if (IS_ERR(c)) {
1040                         err = PTR_ERR(c);
1041                         goto err;
1042                 }
1043
1044                 vm = i915_gem_context_get_vm_rcu(c);
1045
1046                 client[i].scratch[0] = create_scratch(vm, 1024);
1047                 if (IS_ERR(client[i].scratch[0])) {
1048                         err = PTR_ERR(client[i].scratch[0]);
1049                         i915_vm_put(vm);
1050                         kernel_context_close(c);
1051                         goto err;
1052                 }
1053
1054                 client[i].scratch[1] = create_scratch(vm, 1024);
1055                 if (IS_ERR(client[i].scratch[1])) {
1056                         err = PTR_ERR(client[i].scratch[1]);
1057                         i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1058                         i915_vm_put(vm);
1059                         kernel_context_close(c);
1060                         goto err;
1061                 }
1062
1063                 client[i].ctx = c;
1064                 i915_vm_put(vm);
1065         }
1066
1067         for_each_engine(engine, gt, id) {
1068                 if (!engine->kernel_context->vm)
1069                         continue;
1070
1071                 if (!whitelist_writable_count(engine))
1072                         continue;
1073
1074                 /* Read default values */
1075                 err = read_whitelisted_registers(client[0].ctx, engine,
1076                                                  client[0].scratch[0]);
1077                 if (err)
1078                         goto err;
1079
1080                 /* Try to overwrite registers (should only affect ctx0) */
1081                 err = scrub_whitelisted_registers(client[0].ctx, engine);
1082                 if (err)
1083                         goto err;
1084
1085                 /* Read values from ctx1, we expect these to be defaults */
1086                 err = read_whitelisted_registers(client[1].ctx, engine,
1087                                                  client[1].scratch[0]);
1088                 if (err)
1089                         goto err;
1090
1091                 /* Verify that both reads return the same default values */
1092                 err = check_whitelisted_registers(engine,
1093                                                   client[0].scratch[0],
1094                                                   client[1].scratch[0],
1095                                                   result_eq);
1096                 if (err)
1097                         goto err;
1098
1099                 /* Read back the updated values in ctx0 */
1100                 err = read_whitelisted_registers(client[0].ctx, engine,
1101                                                  client[0].scratch[1]);
1102                 if (err)
1103                         goto err;
1104
1105                 /* User should be granted privilege to overwhite regs */
1106                 err = check_whitelisted_registers(engine,
1107                                                   client[0].scratch[0],
1108                                                   client[0].scratch[1],
1109                                                   result_neq);
1110                 if (err)
1111                         goto err;
1112         }
1113
1114 err:
1115         for (i = 0; i < ARRAY_SIZE(client); i++) {
1116                 if (!client[i].ctx)
1117                         break;
1118
1119                 i915_vma_unpin_and_release(&client[i].scratch[1], 0);
1120                 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1121                 kernel_context_close(client[i].ctx);
1122         }
1123
1124         if (igt_flush_test(gt->i915))
1125                 err = -EIO;
1126
1127         return err;
1128 }
1129
1130 static bool
1131 verify_wa_lists(struct i915_gem_context *ctx, struct wa_lists *lists,
1132                 const char *str)
1133 {
1134         struct drm_i915_private *i915 = ctx->i915;
1135         struct i915_gem_engines_iter it;
1136         struct intel_context *ce;
1137         bool ok = true;
1138
1139         ok &= wa_list_verify(&i915->uncore, &lists->gt_wa_list, str);
1140
1141         for_each_gem_engine(ce, i915_gem_context_engines(ctx), it) {
1142                 enum intel_engine_id id = ce->engine->id;
1143
1144                 ok &= engine_wa_list_verify(ce,
1145                                             &lists->engine[id].wa_list,
1146                                             str) == 0;
1147
1148                 ok &= engine_wa_list_verify(ce,
1149                                             &lists->engine[id].ctx_wa_list,
1150                                             str) == 0;
1151         }
1152
1153         return ok;
1154 }
1155
1156 static int
1157 live_gpu_reset_workarounds(void *arg)
1158 {
1159         struct intel_gt *gt = arg;
1160         struct i915_gem_context *ctx;
1161         intel_wakeref_t wakeref;
1162         struct wa_lists lists;
1163         bool ok;
1164
1165         if (!intel_has_gpu_reset(gt))
1166                 return 0;
1167
1168         ctx = kernel_context(gt->i915);
1169         if (IS_ERR(ctx))
1170                 return PTR_ERR(ctx);
1171
1172         i915_gem_context_lock_engines(ctx);
1173
1174         pr_info("Verifying after GPU reset...\n");
1175
1176         igt_global_reset_lock(gt);
1177         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1178
1179         reference_lists_init(gt, &lists);
1180
1181         ok = verify_wa_lists(ctx, &lists, "before reset");
1182         if (!ok)
1183                 goto out;
1184
1185         intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");
1186
1187         ok = verify_wa_lists(ctx, &lists, "after reset");
1188
1189 out:
1190         i915_gem_context_unlock_engines(ctx);
1191         kernel_context_close(ctx);
1192         reference_lists_fini(gt, &lists);
1193         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1194         igt_global_reset_unlock(gt);
1195
1196         return ok ? 0 : -ESRCH;
1197 }
1198
1199 static int
1200 live_engine_reset_workarounds(void *arg)
1201 {
1202         struct intel_gt *gt = arg;
1203         struct i915_gem_engines_iter it;
1204         struct i915_gem_context *ctx;
1205         struct intel_context *ce;
1206         struct igt_spinner spin;
1207         struct i915_request *rq;
1208         intel_wakeref_t wakeref;
1209         struct wa_lists lists;
1210         int ret = 0;
1211
1212         if (!intel_has_reset_engine(gt))
1213                 return 0;
1214
1215         ctx = kernel_context(gt->i915);
1216         if (IS_ERR(ctx))
1217                 return PTR_ERR(ctx);
1218
1219         igt_global_reset_lock(gt);
1220         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1221
1222         reference_lists_init(gt, &lists);
1223
1224         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1225                 struct intel_engine_cs *engine = ce->engine;
1226                 bool ok;
1227
1228                 pr_info("Verifying after %s reset...\n", engine->name);
1229
1230                 ok = verify_wa_lists(ctx, &lists, "before reset");
1231                 if (!ok) {
1232                         ret = -ESRCH;
1233                         goto err;
1234                 }
1235
1236                 intel_engine_reset(engine, "live_workarounds");
1237
1238                 ok = verify_wa_lists(ctx, &lists, "after idle reset");
1239                 if (!ok) {
1240                         ret = -ESRCH;
1241                         goto err;
1242                 }
1243
1244                 ret = igt_spinner_init(&spin, engine->gt);
1245                 if (ret)
1246                         goto err;
1247
1248                 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
1249                 if (IS_ERR(rq)) {
1250                         ret = PTR_ERR(rq);
1251                         igt_spinner_fini(&spin);
1252                         goto err;
1253                 }
1254
1255                 ret = request_add_spin(rq, &spin);
1256                 if (ret) {
1257                         pr_err("Spinner failed to start\n");
1258                         igt_spinner_fini(&spin);
1259                         goto err;
1260                 }
1261
1262                 intel_engine_reset(engine, "live_workarounds");
1263
1264                 igt_spinner_end(&spin);
1265                 igt_spinner_fini(&spin);
1266
1267                 ok = verify_wa_lists(ctx, &lists, "after busy reset");
1268                 if (!ok) {
1269                         ret = -ESRCH;
1270                         goto err;
1271                 }
1272         }
1273 err:
1274         i915_gem_context_unlock_engines(ctx);
1275         reference_lists_fini(gt, &lists);
1276         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1277         igt_global_reset_unlock(gt);
1278         kernel_context_close(ctx);
1279
1280         igt_flush_test(gt->i915);
1281
1282         return ret;
1283 }
1284
1285 int intel_workarounds_live_selftests(struct drm_i915_private *i915)
1286 {
1287         static const struct i915_subtest tests[] = {
1288                 SUBTEST(live_dirty_whitelist),
1289                 SUBTEST(live_reset_whitelist),
1290                 SUBTEST(live_isolated_whitelist),
1291                 SUBTEST(live_gpu_reset_workarounds),
1292                 SUBTEST(live_engine_reset_workarounds),
1293         };
1294
1295         if (intel_gt_is_wedged(&i915->gt))
1296                 return 0;
1297
1298         return intel_gt_live_subtests(tests, &i915->gt);
1299 }