drivers/gpu/drm/i915/selftests/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/list_sort.h>
  26 #include <linux/prime_numbers.h>
  27
  28 #include "gem/i915_gem_context.h"
  29 #include "gem/selftests/mock_context.h"
  30 #include "gt/intel_context.h"
  31
  32 #include "i915_random.h"
  33 #include "i915_selftest.h"
  34
  35 #include "mock_drm.h"
  36 #include "mock_gem_device.h"
  37 #include "mock_gtt.h"
  38 #include "igt_flush_test.h"
  39
  40 static void cleanup_freed_objects(struct drm_i915_private *i915)
  41 {
  42         i915_gem_drain_freed_objects(i915);
  43 }
  44
  45 static void fake_free_pages(struct drm_i915_gem_object *obj,
  46                             struct sg_table *pages)
  47 {
  48         sg_free_table(pages);
  49         kfree(pages);
  50 }
  51
  52 static int fake_get_pages(struct drm_i915_gem_object *obj)
  53 {
  54 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  55 #define PFN_BIAS 0x1000
  56         struct sg_table *pages;
  57         struct scatterlist *sg;
  58         unsigned int sg_page_sizes;
  59         typeof(obj->base.size) rem;
  60
  61         pages = kmalloc(sizeof(*pages), GFP);
  62         if (!pages)
  63                 return -ENOMEM;
  64
  65         rem = round_up(obj->base.size, BIT(31)) >> 31;
  66         if (sg_alloc_table(pages, rem, GFP)) {
  67                 kfree(pages);
  68                 return -ENOMEM;
  69         }
  70
  71         sg_page_sizes = 0;
  72         rem = obj->base.size;
  73         for (sg = pages->sgl; sg; sg = sg_next(sg)) {
  74                 unsigned long len = min_t(typeof(rem), rem, BIT(31));
  75
  76                 GEM_BUG_ON(!len);
  77                 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
  78                 sg_dma_address(sg) = page_to_phys(sg_page(sg));
  79                 sg_dma_len(sg) = len;
  80                 sg_page_sizes |= len;
  81
  82                 rem -= len;
  83         }
  84         GEM_BUG_ON(rem);
  85
  86         __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
  87
  88         return 0;
  89 #undef GFP
  90 }
  91
  92 static void fake_put_pages(struct drm_i915_gem_object *obj,
  93                            struct sg_table *pages)
  94 {
  95         fake_free_pages(obj, pages);
  96         obj->mm.dirty = false;
  97 }
  98
  99 static const struct drm_i915_gem_object_ops fake_ops = {
 100         .name = "fake-gem",
 101         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 102         .get_pages = fake_get_pages,
 103         .put_pages = fake_put_pages,
 104 };
 105
 106 static struct drm_i915_gem_object *
 107 fake_dma_object(struct drm_i915_private *i915, u64 size)
 108 {
 109         static struct lock_class_key lock_class;
 110         struct drm_i915_gem_object *obj;
 111
 112         GEM_BUG_ON(!size);
 113         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 114
 115         if (overflows_type(size, obj->base.size))
 116                 return ERR_PTR(-E2BIG);
 117
 118         obj = i915_gem_object_alloc();
 119         if (!obj)
 120                 goto err;
 121
 122         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 123         i915_gem_object_init(obj, &fake_ops, &lock_class);
 124
 125         i915_gem_object_set_volatile(obj);
 126
 127         obj->write_domain = I915_GEM_DOMAIN_CPU;
 128         obj->read_domains = I915_GEM_DOMAIN_CPU;
 129         obj->cache_level = I915_CACHE_NONE;
 130
 131         /* Preallocate the "backing storage" */
 132         if (i915_gem_object_pin_pages(obj))
 133                 goto err_obj;
 134
 135         i915_gem_object_unpin_pages(obj);
 136         return obj;
 137
 138 err_obj:
 139         i915_gem_object_put(obj);
 140 err:
 141         return ERR_PTR(-ENOMEM);
 142 }
 143
 144 static int igt_ppgtt_alloc(void *arg)
 145 {
 146         struct drm_i915_private *dev_priv = arg;
 147         struct i915_ppgtt *ppgtt;
 148         u64 size, last, limit;
 149         int err = 0;
 150
 151         /* Allocate a ppggt and try to fill the entire range */
 152
 153         if (!HAS_PPGTT(dev_priv))
 154                 return 0;
 155
 156         ppgtt = i915_ppgtt_create(&dev_priv->gt);
 157         if (IS_ERR(ppgtt))
 158                 return PTR_ERR(ppgtt);
 159
 160         if (!ppgtt->vm.allocate_va_range)
 161                 goto err_ppgtt_cleanup;
 162
 163         /*
 164          * While we only allocate the page tables here and so we could
 165          * address a much larger GTT than we could actually fit into
 166          * RAM, a practical limit is the amount of physical pages in the system.
 167          * This should ensure that we do not run into the oomkiller during
 168          * the test and take down the machine wilfully.
 169          */
 170         limit = totalram_pages() << PAGE_SHIFT;
 171         limit = min(ppgtt->vm.total, limit);
 172
 173         /* Check we can allocate the entire range */
 174         for (size = 4096; size <= limit; size <<= 2) {
 175                 struct i915_vm_pt_stash stash = {};
 176
 177                 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size);
 178                 if (err)
 179                         goto err_ppgtt_cleanup;
 180
 181                 err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);
 182                 if (err) {
 183                         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 184                         goto err_ppgtt_cleanup;
 185                 }
 186
 187                 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size);
 188                 cond_resched();
 189
 190                 ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
 191
 192                 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 193         }
 194
 195         /* Check we can incrementally allocate the entire range */
 196         for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
 197                 struct i915_vm_pt_stash stash = {};
 198
 199                 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last);
 200                 if (err)
 201                         goto err_ppgtt_cleanup;
 202
 203                 err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);
 204                 if (err) {
 205                         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 206                         goto err_ppgtt_cleanup;
 207                 }
 208
 209                 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash,
 210                                             last, size - last);
 211                 cond_resched();
 212
 213                 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 214         }
 215
 216 err_ppgtt_cleanup:
 217         i915_vm_put(&ppgtt->vm);
 218         return err;
 219 }
 220
 221 static int lowlevel_hole(struct i915_address_space *vm,
 222                          u64 hole_start, u64 hole_end,
 223                          unsigned long end_time)
 224 {
 225         I915_RND_STATE(seed_prng);
 226         struct i915_vma *mock_vma;
 227         unsigned int size;
 228
 229         mock_vma = kzalloc(sizeof(*mock_vma), GFP_KERNEL);
 230         if (!mock_vma)
 231                 return -ENOMEM;
 232
 233         /* Keep creating larger objects until one cannot fit into the hole */
 234         for (size = 12; (hole_end - hole_start) >> size; size++) {
 235                 I915_RND_SUBSTATE(prng, seed_prng);
 236                 struct drm_i915_gem_object *obj;
 237                 unsigned int *order, count, n;
 238                 u64 hole_size;
 239
 240                 hole_size = (hole_end - hole_start) >> size;
 241                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 242                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 243                 count = hole_size >> 1;
 244                 if (!count) {
 245                         pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
 246                                  __func__, hole_start, hole_end, size, hole_size);
 247                         break;
 248                 }
 249
 250                 do {
 251                         order = i915_random_order(count, &prng);
 252                         if (order)
 253                                 break;
 254                 } while (count >>= 1);
 255                 if (!count) {
 256                         kfree(mock_vma);
 257                         return -ENOMEM;
 258                 }
 259                 GEM_BUG_ON(!order);
 260
 261                 GEM_BUG_ON(count * BIT_ULL(size) > vm->total);
 262                 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end);
 263
 264                 /* Ignore allocation failures (i.e. don't report them as
 265                  * a test failure) as we are purposefully allocating very
 266                  * large objects without checking that we have sufficient
 267                  * memory. We expect to hit -ENOMEM.
 268                  */
 269
 270                 obj = fake_dma_object(vm->i915, BIT_ULL(size));
 271                 if (IS_ERR(obj)) {
 272                         kfree(order);
 273                         break;
 274                 }
 275
 276                 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
 277
 278                 if (i915_gem_object_pin_pages(obj)) {
 279                         i915_gem_object_put(obj);
 280                         kfree(order);
 281                         break;
 282                 }
 283
 284                 for (n = 0; n < count; n++) {
 285                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 286                         intel_wakeref_t wakeref;
 287
 288                         GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
 289
 290                         if (igt_timeout(end_time,
 291                                         "%s timed out before %d/%d\n",
 292                                         __func__, n, count)) {
 293                                 hole_end = hole_start; /* quit */
 294                                 break;
 295                         }
 296
 297                         if (vm->allocate_va_range) {
 298                                 struct i915_vm_pt_stash stash = {};
 299
 300                                 if (i915_vm_alloc_pt_stash(vm, &stash,
 301                                                            BIT_ULL(size)))
 302                                         break;
 303
 304                                 if (i915_vm_pin_pt_stash(vm, &stash)) {
 305                                         i915_vm_free_pt_stash(vm, &stash);
 306                                         break;
 307                                 }
 308
 309                                 vm->allocate_va_range(vm, &stash,
 310                                                       addr, BIT_ULL(size));
 311
 312                                 i915_vm_free_pt_stash(vm, &stash);
 313                         }
 314
 315                         mock_vma->pages = obj->mm.pages;
 316                         mock_vma->node.size = BIT_ULL(size);
 317                         mock_vma->node.start = addr;
 318
 319                         with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
 320                                 vm->insert_entries(vm, mock_vma,
 321                                                    I915_CACHE_NONE, 0);
 322                 }
 323                 count = n;
 324
 325                 i915_random_reorder(order, count, &prng);
 326                 for (n = 0; n < count; n++) {
 327                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 328                         intel_wakeref_t wakeref;
 329
 330                         GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
 331                         with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
 332                                 vm->clear_range(vm, addr, BIT_ULL(size));
 333                 }
 334
 335                 i915_gem_object_unpin_pages(obj);
 336                 i915_gem_object_put(obj);
 337
 338                 kfree(order);
 339
 340                 cleanup_freed_objects(vm->i915);
 341         }
 342
 343         kfree(mock_vma);
 344         return 0;
 345 }
 346
 347 static void close_object_list(struct list_head *objects,
 348                               struct i915_address_space *vm)
 349 {
 350         struct drm_i915_gem_object *obj, *on;
 351         int ignored;
 352
 353         list_for_each_entry_safe(obj, on, objects, st_link) {
 354                 struct i915_vma *vma;
 355
 356                 vma = i915_vma_instance(obj, vm, NULL);
 357                 if (!IS_ERR(vma))
 358                         ignored = i915_vma_unbind(vma);
 359
 360                 list_del(&obj->st_link);
 361                 i915_gem_object_put(obj);
 362         }
 363 }
 364
 365 static int fill_hole(struct i915_address_space *vm,
 366                      u64 hole_start, u64 hole_end,
 367                      unsigned long end_time)
 368 {
 369         const u64 hole_size = hole_end - hole_start;
 370         struct drm_i915_gem_object *obj;
 371         const unsigned long max_pages =
 372                 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT);
 373         const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
 374         unsigned long npages, prime, flags;
 375         struct i915_vma *vma;
 376         LIST_HEAD(objects);
 377         int err;
 378
 379         /* Try binding many VMA working inwards from either edge */
 380
 381         flags = PIN_OFFSET_FIXED | PIN_USER;
 382         if (i915_is_ggtt(vm))
 383                 flags |= PIN_GLOBAL;
 384
 385         for_each_prime_number_from(prime, 2, max_step) {
 386                 for (npages = 1; npages <= max_pages; npages *= prime) {
 387                         const u64 full_size = npages << PAGE_SHIFT;
 388                         const struct {
 389                                 const char *name;
 390                                 u64 offset;
 391                                 int step;
 392                         } phases[] = {
 393                                 { "top-down", hole_end, -1, },
 394                                 { "bottom-up", hole_start, 1, },
 395                                 { }
 396                         }, *p;
 397
 398                         obj = fake_dma_object(vm->i915, full_size);
 399                         if (IS_ERR(obj))
 400                                 break;
 401
 402                         list_add(&obj->st_link, &objects);
 403
 404                         /* Align differing sized objects against the edges, and
 405                          * check we don't walk off into the void when binding
 406                          * them into the GTT.
 407                          */
 408                         for (p = phases; p->name; p++) {
 409                                 u64 offset;
 410
 411                                 offset = p->offset;
 412                                 list_for_each_entry(obj, &objects, st_link) {
 413                                         vma = i915_vma_instance(obj, vm, NULL);
 414                                         if (IS_ERR(vma))
 415                                                 continue;
 416
 417                                         if (p->step < 0) {
 418                                                 if (offset < hole_start + obj->base.size)
 419                                                         break;
 420                                                 offset -= obj->base.size;
 421                                         }
 422
 423                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 424                                         if (err) {
 425                                                 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 426                                                        __func__, p->name, err, npages, prime, offset);
 427                                                 goto err;
 428                                         }
 429
 430                                         if (!drm_mm_node_allocated(&vma->node) ||
 431                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 432                                                 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 433                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 434                                                        offset);
 435                                                 err = -EINVAL;
 436                                                 goto err;
 437                                         }
 438
 439                                         i915_vma_unpin(vma);
 440
 441                                         if (p->step > 0) {
 442                                                 if (offset + obj->base.size > hole_end)
 443                                                         break;
 444                                                 offset += obj->base.size;
 445                                         }
 446                                 }
 447
 448                                 offset = p->offset;
 449                                 list_for_each_entry(obj, &objects, st_link) {
 450                                         vma = i915_vma_instance(obj, vm, NULL);
 451                                         if (IS_ERR(vma))
 452                                                 continue;
 453
 454                                         if (p->step < 0) {
 455                                                 if (offset < hole_start + obj->base.size)
 456                                                         break;
 457                                                 offset -= obj->base.size;
 458                                         }
 459
 460                                         if (!drm_mm_node_allocated(&vma->node) ||
 461                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 462                                                 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
 463                                                        __func__, p->name, vma->node.start, vma->node.size,
 464                                                        offset);
 465                                                 err = -EINVAL;
 466                                                 goto err;
 467                                         }
 468
 469                                         err = i915_vma_unbind(vma);
 470                                         if (err) {
 471                                                 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 472                                                        __func__, p->name, vma->node.start, vma->node.size,
 473                                                        err);
 474                                                 goto err;
 475                                         }
 476
 477                                         if (p->step > 0) {
 478                                                 if (offset + obj->base.size > hole_end)
 479                                                         break;
 480                                                 offset += obj->base.size;
 481                                         }
 482                                 }
 483
 484                                 offset = p->offset;
 485                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 486                                         vma = i915_vma_instance(obj, vm, NULL);
 487                                         if (IS_ERR(vma))
 488                                                 continue;
 489
 490                                         if (p->step < 0) {
 491                                                 if (offset < hole_start + obj->base.size)
 492                                                         break;
 493                                                 offset -= obj->base.size;
 494                                         }
 495
 496                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 497                                         if (err) {
 498                                                 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 499                                                        __func__, p->name, err, npages, prime, offset);
 500                                                 goto err;
 501                                         }
 502
 503                                         if (!drm_mm_node_allocated(&vma->node) ||
 504                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 505                                                 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 506                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 507                                                        offset);
 508                                                 err = -EINVAL;
 509                                                 goto err;
 510                                         }
 511
 512                                         i915_vma_unpin(vma);
 513
 514                                         if (p->step > 0) {
 515                                                 if (offset + obj->base.size > hole_end)
 516                                                         break;
 517                                                 offset += obj->base.size;
 518                                         }
 519                                 }
 520
 521                                 offset = p->offset;
 522                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 523                                         vma = i915_vma_instance(obj, vm, NULL);
 524                                         if (IS_ERR(vma))
 525                                                 continue;
 526
 527                                         if (p->step < 0) {
 528                                                 if (offset < hole_start + obj->base.size)
 529                                                         break;
 530                                                 offset -= obj->base.size;
 531                                         }
 532
 533                                         if (!drm_mm_node_allocated(&vma->node) ||
 534                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 535                                                 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 536                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 537                                                        offset);
 538                                                 err = -EINVAL;
 539                                                 goto err;
 540                                         }
 541
 542                                         err = i915_vma_unbind(vma);
 543                                         if (err) {
 544                                                 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 545                                                        __func__, p->name, vma->node.start, vma->node.size,
 546                                                        err);
 547                                                 goto err;
 548                                         }
 549
 550                                         if (p->step > 0) {
 551                                                 if (offset + obj->base.size > hole_end)
 552                                                         break;
 553                                                 offset += obj->base.size;
 554                                         }
 555                                 }
 556                         }
 557
 558                         if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n",
 559                                         __func__, npages, prime)) {
 560                                 err = -EINTR;
 561                                 goto err;
 562                         }
 563                 }
 564
 565                 close_object_list(&objects, vm);
 566                 cleanup_freed_objects(vm->i915);
 567         }
 568
 569         return 0;
 570
 571 err:
 572         close_object_list(&objects, vm);
 573         return err;
 574 }
 575
 576 static int walk_hole(struct i915_address_space *vm,
 577                      u64 hole_start, u64 hole_end,
 578                      unsigned long end_time)
 579 {
 580         const u64 hole_size = hole_end - hole_start;
 581         const unsigned long max_pages =
 582                 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
 583         unsigned long flags;
 584         u64 size;
 585
 586         /* Try binding a single VMA in different positions within the hole */
 587
 588         flags = PIN_OFFSET_FIXED | PIN_USER;
 589         if (i915_is_ggtt(vm))
 590                 flags |= PIN_GLOBAL;
 591
 592         for_each_prime_number_from(size, 1, max_pages) {
 593                 struct drm_i915_gem_object *obj;
 594                 struct i915_vma *vma;
 595                 u64 addr;
 596                 int err = 0;
 597
 598                 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT);
 599                 if (IS_ERR(obj))
 600                         break;
 601
 602                 vma = i915_vma_instance(obj, vm, NULL);
 603                 if (IS_ERR(vma)) {
 604                         err = PTR_ERR(vma);
 605                         goto err_put;
 606                 }
 607
 608                 for (addr = hole_start;
 609                      addr + obj->base.size < hole_end;
 610                      addr += obj->base.size) {
 611                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 612                         if (err) {
 613                                 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
 614                                        __func__, addr, vma->size,
 615                                        hole_start, hole_end, err);
 616                                 goto err_put;
 617                         }
 618                         i915_vma_unpin(vma);
 619
 620                         if (!drm_mm_node_allocated(&vma->node) ||
 621                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 622                                 pr_err("%s incorrect at %llx + %llx\n",
 623                                        __func__, addr, vma->size);
 624                                 err = -EINVAL;
 625                                 goto err_put;
 626                         }
 627
 628                         err = i915_vma_unbind(vma);
 629                         if (err) {
 630                                 pr_err("%s unbind failed at %llx + %llx  with err=%d\n",
 631                                        __func__, addr, vma->size, err);
 632                                 goto err_put;
 633                         }
 634
 635                         GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
 636
 637                         if (igt_timeout(end_time,
 638                                         "%s timed out at %llx\n",
 639                                         __func__, addr)) {
 640                                 err = -EINTR;
 641                                 goto err_put;
 642                         }
 643                 }
 644
 645 err_put:
 646                 i915_gem_object_put(obj);
 647                 if (err)
 648                         return err;
 649
 650                 cleanup_freed_objects(vm->i915);
 651         }
 652
 653         return 0;
 654 }
 655
 656 static int pot_hole(struct i915_address_space *vm,
 657                     u64 hole_start, u64 hole_end,
 658                     unsigned long end_time)
 659 {
 660         struct drm_i915_gem_object *obj;
 661         struct i915_vma *vma;
 662         unsigned long flags;
 663         unsigned int pot;
 664         int err = 0;
 665
 666         flags = PIN_OFFSET_FIXED | PIN_USER;
 667         if (i915_is_ggtt(vm))
 668                 flags |= PIN_GLOBAL;
 669
 670         obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE);
 671         if (IS_ERR(obj))
 672                 return PTR_ERR(obj);
 673
 674         vma = i915_vma_instance(obj, vm, NULL);
 675         if (IS_ERR(vma)) {
 676                 err = PTR_ERR(vma);
 677                 goto err_obj;
 678         }
 679
 680         /* Insert a pair of pages across every pot boundary within the hole */
 681         for (pot = fls64(hole_end - 1) - 1;
 682              pot > ilog2(2 * I915_GTT_PAGE_SIZE);
 683              pot--) {
 684                 u64 step = BIT_ULL(pot);
 685                 u64 addr;
 686
 687                 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
 688                      addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
 689                      addr += step) {
 690                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 691                         if (err) {
 692                                 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
 693                                        __func__,
 694                                        addr,
 695                                        hole_start, hole_end,
 696                                        err);
 697                                 goto err_obj;
 698                         }
 699
 700                         if (!drm_mm_node_allocated(&vma->node) ||
 701                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 702                                 pr_err("%s incorrect at %llx + %llx\n",
 703                                        __func__, addr, vma->size);
 704                                 i915_vma_unpin(vma);
 705                                 err = i915_vma_unbind(vma);
 706                                 err = -EINVAL;
 707                                 goto err_obj;
 708                         }
 709
 710                         i915_vma_unpin(vma);
 711                         err = i915_vma_unbind(vma);
 712                         GEM_BUG_ON(err);
 713                 }
 714
 715                 if (igt_timeout(end_time,
 716                                 "%s timed out after %d/%d\n",
 717                                 __func__, pot, fls64(hole_end - 1) - 1)) {
 718                         err = -EINTR;
 719                         goto err_obj;
 720                 }
 721         }
 722
 723 err_obj:
 724         i915_gem_object_put(obj);
 725         return err;
 726 }
 727
 728 static int drunk_hole(struct i915_address_space *vm,
 729                       u64 hole_start, u64 hole_end,
 730                       unsigned long end_time)
 731 {
 732         I915_RND_STATE(prng);
 733         unsigned int size;
 734         unsigned long flags;
 735
 736         flags = PIN_OFFSET_FIXED | PIN_USER;
 737         if (i915_is_ggtt(vm))
 738                 flags |= PIN_GLOBAL;
 739
 740         /* Keep creating larger objects until one cannot fit into the hole */
 741         for (size = 12; (hole_end - hole_start) >> size; size++) {
 742                 struct drm_i915_gem_object *obj;
 743                 unsigned int *order, count, n;
 744                 struct i915_vma *vma;
 745                 u64 hole_size;
 746                 int err = -ENODEV;
 747
 748                 hole_size = (hole_end - hole_start) >> size;
 749                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 750                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 751                 count = hole_size >> 1;
 752                 if (!count) {
 753                         pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
 754                                  __func__, hole_start, hole_end, size, hole_size);
 755                         break;
 756                 }
 757
 758                 do {
 759                         order = i915_random_order(count, &prng);
 760                         if (order)
 761                                 break;
 762                 } while (count >>= 1);
 763                 if (!count)
 764                         return -ENOMEM;
 765                 GEM_BUG_ON(!order);
 766
 767                 /* Ignore allocation failures (i.e. don't report them as
 768                  * a test failure) as we are purposefully allocating very
 769                  * large objects without checking that we have sufficient
 770                  * memory. We expect to hit -ENOMEM.
 771                  */
 772
 773                 obj = fake_dma_object(vm->i915, BIT_ULL(size));
 774                 if (IS_ERR(obj)) {
 775                         kfree(order);
 776                         break;
 777                 }
 778
 779                 vma = i915_vma_instance(obj, vm, NULL);
 780                 if (IS_ERR(vma)) {
 781                         err = PTR_ERR(vma);
 782                         goto err_obj;
 783                 }
 784
 785                 GEM_BUG_ON(vma->size != BIT_ULL(size));
 786
 787                 for (n = 0; n < count; n++) {
 788                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 789
 790                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 791                         if (err) {
 792                                 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 793                                        __func__,
 794                                        addr, BIT_ULL(size),
 795                                        hole_start, hole_end,
 796                                        err);
 797                                 goto err_obj;
 798                         }
 799
 800                         if (!drm_mm_node_allocated(&vma->node) ||
 801                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 802                                 pr_err("%s incorrect at %llx + %llx\n",
 803                                        __func__, addr, BIT_ULL(size));
 804                                 i915_vma_unpin(vma);
 805                                 err = i915_vma_unbind(vma);
 806                                 err = -EINVAL;
 807                                 goto err_obj;
 808                         }
 809
 810                         i915_vma_unpin(vma);
 811                         err = i915_vma_unbind(vma);
 812                         GEM_BUG_ON(err);
 813
 814                         if (igt_timeout(end_time,
 815                                         "%s timed out after %d/%d\n",
 816                                         __func__, n, count)) {
 817                                 err = -EINTR;
 818                                 goto err_obj;
 819                         }
 820                 }
 821
 822 err_obj:
 823                 i915_gem_object_put(obj);
 824                 kfree(order);
 825                 if (err)
 826                         return err;
 827
 828                 cleanup_freed_objects(vm->i915);
 829         }
 830
 831         return 0;
 832 }
 833
 834 static int __shrink_hole(struct i915_address_space *vm,
 835                          u64 hole_start, u64 hole_end,
 836                          unsigned long end_time)
 837 {
 838         struct drm_i915_gem_object *obj;
 839         unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
 840         unsigned int order = 12;
 841         LIST_HEAD(objects);
 842         int err = 0;
 843         u64 addr;
 844
 845         /* Keep creating larger objects until one cannot fit into the hole */
 846         for (addr = hole_start; addr < hole_end; ) {
 847                 struct i915_vma *vma;
 848                 u64 size = BIT_ULL(order++);
 849
 850                 size = min(size, hole_end - addr);
 851                 obj = fake_dma_object(vm->i915, size);
 852                 if (IS_ERR(obj)) {
 853                         err = PTR_ERR(obj);
 854                         break;
 855                 }
 856
 857                 list_add(&obj->st_link, &objects);
 858
 859                 vma = i915_vma_instance(obj, vm, NULL);
 860                 if (IS_ERR(vma)) {
 861                         err = PTR_ERR(vma);
 862                         break;
 863                 }
 864
 865                 GEM_BUG_ON(vma->size != size);
 866
 867                 err = i915_vma_pin(vma, 0, 0, addr | flags);
 868                 if (err) {
 869                         pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 870                                __func__, addr, size, hole_start, hole_end, err);
 871                         break;
 872                 }
 873
 874                 if (!drm_mm_node_allocated(&vma->node) ||
 875                     i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 876                         pr_err("%s incorrect at %llx + %llx\n",
 877                                __func__, addr, size);
 878                         i915_vma_unpin(vma);
 879                         err = i915_vma_unbind(vma);
 880                         err = -EINVAL;
 881                         break;
 882                 }
 883
 884                 i915_vma_unpin(vma);
 885                 addr += size;
 886
 887                 /*
 888                  * Since we are injecting allocation faults at random intervals,
 889                  * wait for this allocation to complete before we change the
 890                  * faultinjection.
 891                  */
 892                 err = i915_vma_sync(vma);
 893                 if (err)
 894                         break;
 895
 896                 if (igt_timeout(end_time,
 897                                 "%s timed out at ofset %llx [%llx - %llx]\n",
 898                                 __func__, addr, hole_start, hole_end)) {
 899                         err = -EINTR;
 900                         break;
 901                 }
 902         }
 903
 904         close_object_list(&objects, vm);
 905         cleanup_freed_objects(vm->i915);
 906         return err;
 907 }
 908
 909 static int shrink_hole(struct i915_address_space *vm,
 910                        u64 hole_start, u64 hole_end,
 911                        unsigned long end_time)
 912 {
 913         unsigned long prime;
 914         int err;
 915
 916         vm->fault_attr.probability = 999;
 917         atomic_set(&vm->fault_attr.times, -1);
 918
 919         for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
 920                 vm->fault_attr.interval = prime;
 921                 err = __shrink_hole(vm, hole_start, hole_end, end_time);
 922                 if (err)
 923                         break;
 924         }
 925
 926         memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
 927
 928         return err;
 929 }
 930
 931 static int shrink_boom(struct i915_address_space *vm,
 932                        u64 hole_start, u64 hole_end,
 933                        unsigned long end_time)
 934 {
 935         unsigned int sizes[] = { SZ_2M, SZ_1G };
 936         struct drm_i915_gem_object *purge;
 937         struct drm_i915_gem_object *explode;
 938         int err;
 939         int i;
 940
 941         /*
 942          * Catch the case which shrink_hole seems to miss. The setup here
 943          * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while
 944          * ensuring that all vma assiocated with the respective pd/pdp are
 945          * unpinned at the time.
 946          */
 947
 948         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
 949                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 950                 unsigned int size = sizes[i];
 951                 struct i915_vma *vma;
 952
 953                 purge = fake_dma_object(vm->i915, size);
 954                 if (IS_ERR(purge))
 955                         return PTR_ERR(purge);
 956
 957                 vma = i915_vma_instance(purge, vm, NULL);
 958                 if (IS_ERR(vma)) {
 959                         err = PTR_ERR(vma);
 960                         goto err_purge;
 961                 }
 962
 963                 err = i915_vma_pin(vma, 0, 0, flags);
 964                 if (err)
 965                         goto err_purge;
 966
 967                 /* Should now be ripe for purging */
 968                 i915_vma_unpin(vma);
 969
 970                 explode = fake_dma_object(vm->i915, size);
 971                 if (IS_ERR(explode)) {
 972                         err = PTR_ERR(explode);
 973                         goto err_purge;
 974                 }
 975
 976                 vm->fault_attr.probability = 100;
 977                 vm->fault_attr.interval = 1;
 978                 atomic_set(&vm->fault_attr.times, -1);
 979
 980                 vma = i915_vma_instance(explode, vm, NULL);
 981                 if (IS_ERR(vma)) {
 982                         err = PTR_ERR(vma);
 983                         goto err_explode;
 984                 }
 985
 986                 err = i915_vma_pin(vma, 0, 0, flags | size);
 987                 if (err)
 988                         goto err_explode;
 989
 990                 i915_vma_unpin(vma);
 991
 992                 i915_gem_object_put(purge);
 993                 i915_gem_object_put(explode);
 994
 995                 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
 996                 cleanup_freed_objects(vm->i915);
 997         }
 998
 999         return 0;
1000
1001 err_explode:
1002         i915_gem_object_put(explode);
1003 err_purge:
1004         i915_gem_object_put(purge);
1005         memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1006         return err;
1007 }
1008
1009 static int exercise_ppgtt(struct drm_i915_private *dev_priv,
1010                           int (*func)(struct i915_address_space *vm,
1011                                       u64 hole_start, u64 hole_end,
1012                                       unsigned long end_time))
1013 {
1014         struct i915_ppgtt *ppgtt;
1015         IGT_TIMEOUT(end_time);
1016         struct file *file;
1017         int err;
1018
1019         if (!HAS_FULL_PPGTT(dev_priv))
1020                 return 0;
1021
1022         file = mock_file(dev_priv);
1023         if (IS_ERR(file))
1024                 return PTR_ERR(file);
1025
1026         ppgtt = i915_ppgtt_create(&dev_priv->gt);
1027         if (IS_ERR(ppgtt)) {
1028                 err = PTR_ERR(ppgtt);
1029                 goto out_free;
1030         }
1031         GEM_BUG_ON(offset_in_page(ppgtt->vm.total));
1032         GEM_BUG_ON(!atomic_read(&ppgtt->vm.open));
1033
1034         err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time);
1035
1036         i915_vm_put(&ppgtt->vm);
1037
1038 out_free:
1039         fput(file);
1040         return err;
1041 }
1042
1043 static int igt_ppgtt_fill(void *arg)
1044 {
1045         return exercise_ppgtt(arg, fill_hole);
1046 }
1047
1048 static int igt_ppgtt_walk(void *arg)
1049 {
1050         return exercise_ppgtt(arg, walk_hole);
1051 }
1052
1053 static int igt_ppgtt_pot(void *arg)
1054 {
1055         return exercise_ppgtt(arg, pot_hole);
1056 }
1057
1058 static int igt_ppgtt_drunk(void *arg)
1059 {
1060         return exercise_ppgtt(arg, drunk_hole);
1061 }
1062
1063 static int igt_ppgtt_lowlevel(void *arg)
1064 {
1065         return exercise_ppgtt(arg, lowlevel_hole);
1066 }
1067
1068 static int igt_ppgtt_shrink(void *arg)
1069 {
1070         return exercise_ppgtt(arg, shrink_hole);
1071 }
1072
1073 static int igt_ppgtt_shrink_boom(void *arg)
1074 {
1075         return exercise_ppgtt(arg, shrink_boom);
1076 }
1077
1078 static int sort_holes(void *priv, const struct list_head *A,
1079                       const struct list_head *B)
1080 {
1081         struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
1082         struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);
1083
1084         if (a->start < b->start)
1085                 return -1;
1086         else
1087                 return 1;
1088 }
1089
1090 static int exercise_ggtt(struct drm_i915_private *i915,
1091                          int (*func)(struct i915_address_space *vm,
1092                                      u64 hole_start, u64 hole_end,
1093                                      unsigned long end_time))
1094 {
1095         struct i915_ggtt *ggtt = &i915->ggtt;
1096         u64 hole_start, hole_end, last = 0;
1097         struct drm_mm_node *node;
1098         IGT_TIMEOUT(end_time);
1099         int err = 0;
1100
1101 restart:
1102         list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes);
1103         drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) {
1104                 if (hole_start < last)
1105                         continue;
1106
1107                 if (ggtt->vm.mm.color_adjust)
1108                         ggtt->vm.mm.color_adjust(node, 0,
1109                                                  &hole_start, &hole_end);
1110                 if (hole_start >= hole_end)
1111                         continue;
1112
1113                 err = func(&ggtt->vm, hole_start, hole_end, end_time);
1114                 if (err)
1115                         break;
1116
1117                 /* As we have manipulated the drm_mm, the list may be corrupt */
1118                 last = hole_end;
1119                 goto restart;
1120         }
1121
1122         return err;
1123 }
1124
1125 static int igt_ggtt_fill(void *arg)
1126 {
1127         return exercise_ggtt(arg, fill_hole);
1128 }
1129
1130 static int igt_ggtt_walk(void *arg)
1131 {
1132         return exercise_ggtt(arg, walk_hole);
1133 }
1134
1135 static int igt_ggtt_pot(void *arg)
1136 {
1137         return exercise_ggtt(arg, pot_hole);
1138 }
1139
1140 static int igt_ggtt_drunk(void *arg)
1141 {
1142         return exercise_ggtt(arg, drunk_hole);
1143 }
1144
1145 static int igt_ggtt_lowlevel(void *arg)
1146 {
1147         return exercise_ggtt(arg, lowlevel_hole);
1148 }
1149
1150 static int igt_ggtt_page(void *arg)
1151 {
1152         const unsigned int count = PAGE_SIZE/sizeof(u32);
1153         I915_RND_STATE(prng);
1154         struct drm_i915_private *i915 = arg;
1155         struct i915_ggtt *ggtt = &i915->ggtt;
1156         struct drm_i915_gem_object *obj;
1157         intel_wakeref_t wakeref;
1158         struct drm_mm_node tmp;
1159         unsigned int *order, n;
1160         int err;
1161
1162         if (!i915_ggtt_has_aperture(ggtt))
1163                 return 0;
1164
1165         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1166         if (IS_ERR(obj))
1167                 return PTR_ERR(obj);
1168
1169         err = i915_gem_object_pin_pages(obj);
1170         if (err)
1171                 goto out_free;
1172
1173         memset(&tmp, 0, sizeof(tmp));
1174         mutex_lock(&ggtt->vm.mutex);
1175         err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp,
1176                                           count * PAGE_SIZE, 0,
1177                                           I915_COLOR_UNEVICTABLE,
1178                                           0, ggtt->mappable_end,
1179                                           DRM_MM_INSERT_LOW);
1180         mutex_unlock(&ggtt->vm.mutex);
1181         if (err)
1182                 goto out_unpin;
1183
1184         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1185
1186         for (n = 0; n < count; n++) {
1187                 u64 offset = tmp.start + n * PAGE_SIZE;
1188
1189                 ggtt->vm.insert_page(&ggtt->vm,
1190                                      i915_gem_object_get_dma_address(obj, 0),
1191                                      offset, I915_CACHE_NONE, 0);
1192         }
1193
1194         order = i915_random_order(count, &prng);
1195         if (!order) {
1196                 err = -ENOMEM;
1197                 goto out_remove;
1198         }
1199
1200         for (n = 0; n < count; n++) {
1201                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1202                 u32 __iomem *vaddr;
1203
1204                 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1205                 iowrite32(n, vaddr + n);
1206                 io_mapping_unmap_atomic(vaddr);
1207         }
1208         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1209
1210         i915_random_reorder(order, count, &prng);
1211         for (n = 0; n < count; n++) {
1212                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1213                 u32 __iomem *vaddr;
1214                 u32 val;
1215
1216                 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1217                 val = ioread32(vaddr + n);
1218                 io_mapping_unmap_atomic(vaddr);
1219
1220                 if (val != n) {
1221                         pr_err("insert page failed: found %d, expected %d\n",
1222                                val, n);
1223                         err = -EINVAL;
1224                         break;
1225                 }
1226         }
1227
1228         kfree(order);
1229 out_remove:
1230         ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size);
1231         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1232         mutex_lock(&ggtt->vm.mutex);
1233         drm_mm_remove_node(&tmp);
1234         mutex_unlock(&ggtt->vm.mutex);
1235 out_unpin:
1236         i915_gem_object_unpin_pages(obj);
1237 out_free:
1238         i915_gem_object_put(obj);
1239         return err;
1240 }
1241
1242 static void track_vma_bind(struct i915_vma *vma)
1243 {
1244         struct drm_i915_gem_object *obj = vma->obj;
1245
1246         __i915_gem_object_pin_pages(obj);
1247
1248         GEM_BUG_ON(vma->pages);
1249         atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE);
1250         __i915_gem_object_pin_pages(obj);
1251         vma->pages = obj->mm.pages;
1252
1253         mutex_lock(&vma->vm->mutex);
1254         list_add_tail(&vma->vm_link, &vma->vm->bound_list);
1255         mutex_unlock(&vma->vm->mutex);
1256 }
1257
1258 static int exercise_mock(struct drm_i915_private *i915,
1259                          int (*func)(struct i915_address_space *vm,
1260                                      u64 hole_start, u64 hole_end,
1261                                      unsigned long end_time))
1262 {
1263         const u64 limit = totalram_pages() << PAGE_SHIFT;
1264         struct i915_address_space *vm;
1265         struct i915_gem_context *ctx;
1266         IGT_TIMEOUT(end_time);
1267         int err;
1268
1269         ctx = mock_context(i915, "mock");
1270         if (!ctx)
1271                 return -ENOMEM;
1272
1273         vm = i915_gem_context_get_vm_rcu(ctx);
1274         err = func(vm, 0, min(vm->total, limit), end_time);
1275         i915_vm_put(vm);
1276
1277         mock_context_close(ctx);
1278         return err;
1279 }
1280
1281 static int igt_mock_fill(void *arg)
1282 {
1283         struct i915_ggtt *ggtt = arg;
1284
1285         return exercise_mock(ggtt->vm.i915, fill_hole);
1286 }
1287
1288 static int igt_mock_walk(void *arg)
1289 {
1290         struct i915_ggtt *ggtt = arg;
1291
1292         return exercise_mock(ggtt->vm.i915, walk_hole);
1293 }
1294
1295 static int igt_mock_pot(void *arg)
1296 {
1297         struct i915_ggtt *ggtt = arg;
1298
1299         return exercise_mock(ggtt->vm.i915, pot_hole);
1300 }
1301
1302 static int igt_mock_drunk(void *arg)
1303 {
1304         struct i915_ggtt *ggtt = arg;
1305
1306         return exercise_mock(ggtt->vm.i915, drunk_hole);
1307 }
1308
1309 static int igt_gtt_reserve(void *arg)
1310 {
1311         struct i915_ggtt *ggtt = arg;
1312         struct drm_i915_gem_object *obj, *on;
1313         I915_RND_STATE(prng);
1314         LIST_HEAD(objects);
1315         u64 total;
1316         int err = -ENODEV;
1317
1318         /* i915_gem_gtt_reserve() tries to reserve the precise range
1319          * for the node, and evicts if it has to. So our test checks that
1320          * it can give us the requsted space and prevent overlaps.
1321          */
1322
1323         /* Start by filling the GGTT */
1324         for (total = 0;
1325              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1326              total += 2 * I915_GTT_PAGE_SIZE) {
1327                 struct i915_vma *vma;
1328
1329                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1330                                                       2 * PAGE_SIZE);
1331                 if (IS_ERR(obj)) {
1332                         err = PTR_ERR(obj);
1333                         goto out;
1334                 }
1335
1336                 err = i915_gem_object_pin_pages(obj);
1337                 if (err) {
1338                         i915_gem_object_put(obj);
1339                         goto out;
1340                 }
1341
1342                 list_add(&obj->st_link, &objects);
1343
1344                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1345                 if (IS_ERR(vma)) {
1346                         err = PTR_ERR(vma);
1347                         goto out;
1348                 }
1349
1350                 mutex_lock(&ggtt->vm.mutex);
1351                 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1352                                            obj->base.size,
1353                                            total,
1354                                            obj->cache_level,
1355                                            0);
1356                 mutex_unlock(&ggtt->vm.mutex);
1357                 if (err) {
1358                         pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1359                                total, ggtt->vm.total, err);
1360                         goto out;
1361                 }
1362                 track_vma_bind(vma);
1363
1364                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1365                 if (vma->node.start != total ||
1366                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1367                         pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1368                                vma->node.start, vma->node.size,
1369                                total, 2*I915_GTT_PAGE_SIZE);
1370                         err = -EINVAL;
1371                         goto out;
1372                 }
1373         }
1374
1375         /* Now we start forcing evictions */
1376         for (total = I915_GTT_PAGE_SIZE;
1377              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1378              total += 2 * I915_GTT_PAGE_SIZE) {
1379                 struct i915_vma *vma;
1380
1381                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1382                                                       2 * PAGE_SIZE);
1383                 if (IS_ERR(obj)) {
1384                         err = PTR_ERR(obj);
1385                         goto out;
1386                 }
1387
1388                 err = i915_gem_object_pin_pages(obj);
1389                 if (err) {
1390                         i915_gem_object_put(obj);
1391                         goto out;
1392                 }
1393
1394                 list_add(&obj->st_link, &objects);
1395
1396                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1397                 if (IS_ERR(vma)) {
1398                         err = PTR_ERR(vma);
1399                         goto out;
1400                 }
1401
1402                 mutex_lock(&ggtt->vm.mutex);
1403                 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1404                                            obj->base.size,
1405                                            total,
1406                                            obj->cache_level,
1407                                            0);
1408                 mutex_unlock(&ggtt->vm.mutex);
1409                 if (err) {
1410                         pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1411                                total, ggtt->vm.total, err);
1412                         goto out;
1413                 }
1414                 track_vma_bind(vma);
1415
1416                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1417                 if (vma->node.start != total ||
1418                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1419                         pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1420                                vma->node.start, vma->node.size,
1421                                total, 2*I915_GTT_PAGE_SIZE);
1422                         err = -EINVAL;
1423                         goto out;
1424                 }
1425         }
1426
1427         /* And then try at random */
1428         list_for_each_entry_safe(obj, on, &objects, st_link) {
1429                 struct i915_vma *vma;
1430                 u64 offset;
1431
1432                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1433                 if (IS_ERR(vma)) {
1434                         err = PTR_ERR(vma);
1435                         goto out;
1436                 }
1437
1438                 err = i915_vma_unbind(vma);
1439                 if (err) {
1440                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1441                         goto out;
1442                 }
1443
1444                 offset = igt_random_offset(&prng,
1445                                            0, ggtt->vm.total,
1446                                            2 * I915_GTT_PAGE_SIZE,
1447                                            I915_GTT_MIN_ALIGNMENT);
1448
1449                 mutex_lock(&ggtt->vm.mutex);
1450                 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1451                                            obj->base.size,
1452                                            offset,
1453                                            obj->cache_level,
1454                                            0);
1455                 mutex_unlock(&ggtt->vm.mutex);
1456                 if (err) {
1457                         pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1458                                total, ggtt->vm.total, err);
1459                         goto out;
1460                 }
1461                 track_vma_bind(vma);
1462
1463                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1464                 if (vma->node.start != offset ||
1465                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1466                         pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1467                                vma->node.start, vma->node.size,
1468                                offset, 2*I915_GTT_PAGE_SIZE);
1469                         err = -EINVAL;
1470                         goto out;
1471                 }
1472         }
1473
1474 out:
1475         list_for_each_entry_safe(obj, on, &objects, st_link) {
1476                 i915_gem_object_unpin_pages(obj);
1477                 i915_gem_object_put(obj);
1478         }
1479         return err;
1480 }
1481
1482 static int igt_gtt_insert(void *arg)
1483 {
1484         struct i915_ggtt *ggtt = arg;
1485         struct drm_i915_gem_object *obj, *on;
1486         struct drm_mm_node tmp = {};
1487         const struct invalid_insert {
1488                 u64 size;
1489                 u64 alignment;
1490                 u64 start, end;
1491         } invalid_insert[] = {
1492                 {
1493                         ggtt->vm.total + I915_GTT_PAGE_SIZE, 0,
1494                         0, ggtt->vm.total,
1495                 },
1496                 {
1497                         2*I915_GTT_PAGE_SIZE, 0,
1498                         0, I915_GTT_PAGE_SIZE,
1499                 },
1500                 {
1501                         -(u64)I915_GTT_PAGE_SIZE, 0,
1502                         0, 4*I915_GTT_PAGE_SIZE,
1503                 },
1504                 {
1505                         -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE,
1506                         0, 4*I915_GTT_PAGE_SIZE,
1507                 },
1508                 {
1509                         I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1,
1510                         I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1,
1511                 },
1512                 {}
1513         }, *ii;
1514         LIST_HEAD(objects);
1515         u64 total;
1516         int err = -ENODEV;
1517
1518         /* i915_gem_gtt_insert() tries to allocate some free space in the GTT
1519          * to the node, evicting if required.
1520          */
1521
1522         /* Check a couple of obviously invalid requests */
1523         for (ii = invalid_insert; ii->size; ii++) {
1524                 mutex_lock(&ggtt->vm.mutex);
1525                 err = i915_gem_gtt_insert(&ggtt->vm, &tmp,
1526                                           ii->size, ii->alignment,
1527                                           I915_COLOR_UNEVICTABLE,
1528                                           ii->start, ii->end,
1529                                           0);
1530                 mutex_unlock(&ggtt->vm.mutex);
1531                 if (err != -ENOSPC) {
1532                         pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1533                                ii->size, ii->alignment, ii->start, ii->end,
1534                                err);
1535                         return -EINVAL;
1536                 }
1537         }
1538
1539         /* Start by filling the GGTT */
1540         for (total = 0;
1541              total + I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1542              total += I915_GTT_PAGE_SIZE) {
1543                 struct i915_vma *vma;
1544
1545                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1546                                                       I915_GTT_PAGE_SIZE);
1547                 if (IS_ERR(obj)) {
1548                         err = PTR_ERR(obj);
1549                         goto out;
1550                 }
1551
1552                 err = i915_gem_object_pin_pages(obj);
1553                 if (err) {
1554                         i915_gem_object_put(obj);
1555                         goto out;
1556                 }
1557
1558                 list_add(&obj->st_link, &objects);
1559
1560                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1561                 if (IS_ERR(vma)) {
1562                         err = PTR_ERR(vma);
1563                         goto out;
1564                 }
1565
1566                 mutex_lock(&ggtt->vm.mutex);
1567                 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1568                                           obj->base.size, 0, obj->cache_level,
1569                                           0, ggtt->vm.total,
1570                                           0);
1571                 mutex_unlock(&ggtt->vm.mutex);
1572                 if (err == -ENOSPC) {
1573                         /* maxed out the GGTT space */
1574                         i915_gem_object_put(obj);
1575                         break;
1576                 }
1577                 if (err) {
1578                         pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1579                                total, ggtt->vm.total, err);
1580                         goto out;
1581                 }
1582                 track_vma_bind(vma);
1583                 __i915_vma_pin(vma);
1584
1585                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1586         }
1587
1588         list_for_each_entry(obj, &objects, st_link) {
1589                 struct i915_vma *vma;
1590
1591                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1592                 if (IS_ERR(vma)) {
1593                         err = PTR_ERR(vma);
1594                         goto out;
1595                 }
1596
1597                 if (!drm_mm_node_allocated(&vma->node)) {
1598                         pr_err("VMA was unexpectedly evicted!\n");
1599                         err = -EINVAL;
1600                         goto out;
1601                 }
1602
1603                 __i915_vma_unpin(vma);
1604         }
1605
1606         /* If we then reinsert, we should find the same hole */
1607         list_for_each_entry_safe(obj, on, &objects, st_link) {
1608                 struct i915_vma *vma;
1609                 u64 offset;
1610
1611                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1612                 if (IS_ERR(vma)) {
1613                         err = PTR_ERR(vma);
1614                         goto out;
1615                 }
1616
1617                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1618                 offset = vma->node.start;
1619
1620                 err = i915_vma_unbind(vma);
1621                 if (err) {
1622                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1623                         goto out;
1624                 }
1625
1626                 mutex_lock(&ggtt->vm.mutex);
1627                 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1628                                           obj->base.size, 0, obj->cache_level,
1629                                           0, ggtt->vm.total,
1630                                           0);
1631                 mutex_unlock(&ggtt->vm.mutex);
1632                 if (err) {
1633                         pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1634                                total, ggtt->vm.total, err);
1635                         goto out;
1636                 }
1637                 track_vma_bind(vma);
1638
1639                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1640                 if (vma->node.start != offset) {
1641                         pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n",
1642                                offset, vma->node.start);
1643                         err = -EINVAL;
1644                         goto out;
1645                 }
1646         }
1647
1648         /* And then force evictions */
1649         for (total = 0;
1650              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1651              total += 2 * I915_GTT_PAGE_SIZE) {
1652                 struct i915_vma *vma;
1653
1654                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1655                                                       2 * I915_GTT_PAGE_SIZE);
1656                 if (IS_ERR(obj)) {
1657                         err = PTR_ERR(obj);
1658                         goto out;
1659                 }
1660
1661                 err = i915_gem_object_pin_pages(obj);
1662                 if (err) {
1663                         i915_gem_object_put(obj);
1664                         goto out;
1665                 }
1666
1667                 list_add(&obj->st_link, &objects);
1668
1669                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1670                 if (IS_ERR(vma)) {
1671                         err = PTR_ERR(vma);
1672                         goto out;
1673                 }
1674
1675                 mutex_lock(&ggtt->vm.mutex);
1676                 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1677                                           obj->base.size, 0, obj->cache_level,
1678                                           0, ggtt->vm.total,
1679                                           0);
1680                 mutex_unlock(&ggtt->vm.mutex);
1681                 if (err) {
1682                         pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1683                                total, ggtt->vm.total, err);
1684                         goto out;
1685                 }
1686                 track_vma_bind(vma);
1687
1688                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1689         }
1690
1691 out:
1692         list_for_each_entry_safe(obj, on, &objects, st_link) {
1693                 i915_gem_object_unpin_pages(obj);
1694                 i915_gem_object_put(obj);
1695         }
1696         return err;
1697 }
1698
1699 int i915_gem_gtt_mock_selftests(void)
1700 {
1701         static const struct i915_subtest tests[] = {
1702                 SUBTEST(igt_mock_drunk),
1703                 SUBTEST(igt_mock_walk),
1704                 SUBTEST(igt_mock_pot),
1705                 SUBTEST(igt_mock_fill),
1706                 SUBTEST(igt_gtt_reserve),
1707                 SUBTEST(igt_gtt_insert),
1708         };
1709         struct drm_i915_private *i915;
1710         struct i915_ggtt *ggtt;
1711         int err;
1712
1713         i915 = mock_gem_device();
1714         if (!i915)
1715                 return -ENOMEM;
1716
1717         ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL);
1718         if (!ggtt) {
1719                 err = -ENOMEM;
1720                 goto out_put;
1721         }
1722         mock_init_ggtt(i915, ggtt);
1723
1724         err = i915_subtests(tests, ggtt);
1725
1726         mock_device_flush(i915);
1727         i915_gem_drain_freed_objects(i915);
1728         mock_fini_ggtt(ggtt);
1729         kfree(ggtt);
1730 out_put:
1731         mock_destroy_device(i915);
1732         return err;
1733 }
1734
1735 static int context_sync(struct intel_context *ce)
1736 {
1737         struct i915_request *rq;
1738         long timeout;
1739
1740         rq = intel_context_create_request(ce);
1741         if (IS_ERR(rq))
1742                 return PTR_ERR(rq);
1743
1744         i915_request_get(rq);
1745         i915_request_add(rq);
1746
1747         timeout = i915_request_wait(rq, 0, HZ / 5);
1748         i915_request_put(rq);
1749
1750         return timeout < 0 ? -EIO : 0;
1751 }
1752
1753 static struct i915_request *
1754 submit_batch(struct intel_context *ce, u64 addr)
1755 {
1756         struct i915_request *rq;
1757         int err;
1758
1759         rq = intel_context_create_request(ce);
1760         if (IS_ERR(rq))
1761                 return rq;
1762
1763         err = 0;
1764         if (rq->engine->emit_init_breadcrumb) /* detect a hang */
1765                 err = rq->engine->emit_init_breadcrumb(rq);
1766         if (err == 0)
1767                 err = rq->engine->emit_bb_start(rq, addr, 0, 0);
1768
1769         if (err == 0)
1770                 i915_request_get(rq);
1771         i915_request_add(rq);
1772
1773         return err ? ERR_PTR(err) : rq;
1774 }
1775
1776 static u32 *spinner(u32 *batch, int i)
1777 {
1778         return batch + i * 64 / sizeof(*batch) + 4;
1779 }
1780
1781 static void end_spin(u32 *batch, int i)
1782 {
1783         *spinner(batch, i) = MI_BATCH_BUFFER_END;
1784         wmb();
1785 }
1786
1787 static int igt_cs_tlb(void *arg)
1788 {
1789         const unsigned int count = PAGE_SIZE / 64;
1790         const unsigned int chunk_size = count * PAGE_SIZE;
1791         struct drm_i915_private *i915 = arg;
1792         struct drm_i915_gem_object *bbe, *act, *out;
1793         struct i915_gem_engines_iter it;
1794         struct i915_address_space *vm;
1795         struct i915_gem_context *ctx;
1796         struct intel_context *ce;
1797         struct i915_vma *vma;
1798         I915_RND_STATE(prng);
1799         struct file *file;
1800         unsigned int i;
1801         u32 *result;
1802         u32 *batch;
1803         int err = 0;
1804
1805         /*
1806          * Our mission here is to fool the hardware to execute something
1807          * from scratch as it has not seen the batch move (due to missing
1808          * the TLB invalidate).
1809          */
1810
1811         file = mock_file(i915);
1812         if (IS_ERR(file))
1813                 return PTR_ERR(file);
1814
1815         ctx = live_context(i915, file);
1816         if (IS_ERR(ctx)) {
1817                 err = PTR_ERR(ctx);
1818                 goto out_unlock;
1819         }
1820
1821         vm = i915_gem_context_get_vm_rcu(ctx);
1822         if (i915_is_ggtt(vm))
1823                 goto out_vm;
1824
1825         /* Create two pages; dummy we prefill the TLB, and intended */
1826         bbe = i915_gem_object_create_internal(i915, PAGE_SIZE);
1827         if (IS_ERR(bbe)) {
1828                 err = PTR_ERR(bbe);
1829                 goto out_vm;
1830         }
1831
1832         batch = i915_gem_object_pin_map(bbe, I915_MAP_WC);
1833         if (IS_ERR(batch)) {
1834                 err = PTR_ERR(batch);
1835                 goto out_put_bbe;
1836         }
1837         memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32));
1838         i915_gem_object_flush_map(bbe);
1839         i915_gem_object_unpin_map(bbe);
1840
1841         act = i915_gem_object_create_internal(i915, PAGE_SIZE);
1842         if (IS_ERR(act)) {
1843                 err = PTR_ERR(act);
1844                 goto out_put_bbe;
1845         }
1846
1847         /* Track the execution of each request by writing into different slot */
1848         batch = i915_gem_object_pin_map(act, I915_MAP_WC);
1849         if (IS_ERR(batch)) {
1850                 err = PTR_ERR(batch);
1851                 goto out_put_act;
1852         }
1853         for (i = 0; i < count; i++) {
1854                 u32 *cs = batch + i * 64 / sizeof(*cs);
1855                 u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32);
1856
1857                 GEM_BUG_ON(INTEL_GEN(i915) < 6);
1858                 cs[0] = MI_STORE_DWORD_IMM_GEN4;
1859                 if (INTEL_GEN(i915) >= 8) {
1860                         cs[1] = lower_32_bits(addr);
1861                         cs[2] = upper_32_bits(addr);
1862                         cs[3] = i;
1863                         cs[4] = MI_NOOP;
1864                         cs[5] = MI_BATCH_BUFFER_START_GEN8;
1865                 } else {
1866                         cs[1] = 0;
1867                         cs[2] = lower_32_bits(addr);
1868                         cs[3] = i;
1869                         cs[4] = MI_NOOP;
1870                         cs[5] = MI_BATCH_BUFFER_START;
1871                 }
1872         }
1873
1874         out = i915_gem_object_create_internal(i915, PAGE_SIZE);
1875         if (IS_ERR(out)) {
1876                 err = PTR_ERR(out);
1877                 goto out_put_batch;
1878         }
1879         i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED);
1880
1881         vma = i915_vma_instance(out, vm, NULL);
1882         if (IS_ERR(vma)) {
1883                 err = PTR_ERR(vma);
1884                 goto out_put_out;
1885         }
1886
1887         err = i915_vma_pin(vma, 0, 0,
1888                            PIN_USER |
1889                            PIN_OFFSET_FIXED |
1890                            (vm->total - PAGE_SIZE));
1891         if (err)
1892                 goto out_put_out;
1893         GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE);
1894
1895         result = i915_gem_object_pin_map(out, I915_MAP_WB);
1896         if (IS_ERR(result)) {
1897                 err = PTR_ERR(result);
1898                 goto out_put_out;
1899         }
1900
1901         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1902                 IGT_TIMEOUT(end_time);
1903                 unsigned long pass = 0;
1904
1905                 if (!intel_engine_can_store_dword(ce->engine))
1906                         continue;
1907
1908                 while (!__igt_timeout(end_time, NULL)) {
1909                         struct i915_vm_pt_stash stash = {};
1910                         struct i915_request *rq;
1911                         u64 offset;
1912
1913                         offset = igt_random_offset(&prng,
1914                                                    0, vm->total - PAGE_SIZE,
1915                                                    chunk_size, PAGE_SIZE);
1916
1917                         memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32));
1918
1919                         vma = i915_vma_instance(bbe, vm, NULL);
1920                         if (IS_ERR(vma)) {
1921                                 err = PTR_ERR(vma);
1922                                 goto end;
1923                         }
1924
1925                         err = vma->ops->set_pages(vma);
1926                         if (err)
1927                                 goto end;
1928
1929                         err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size);
1930                         if (err)
1931                                 goto end;
1932
1933                         err = i915_vm_pin_pt_stash(vm, &stash);
1934                         if (err) {
1935                                 i915_vm_free_pt_stash(vm, &stash);
1936                                 goto end;
1937                         }
1938
1939                         vm->allocate_va_range(vm, &stash, offset, chunk_size);
1940
1941                         i915_vm_free_pt_stash(vm, &stash);
1942
1943                         /* Prime the TLB with the dummy pages */
1944                         for (i = 0; i < count; i++) {
1945                                 vma->node.start = offset + i * PAGE_SIZE;
1946                                 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
1947
1948                                 rq = submit_batch(ce, vma->node.start);
1949                                 if (IS_ERR(rq)) {
1950                                         err = PTR_ERR(rq);
1951                                         goto end;
1952                                 }
1953                                 i915_request_put(rq);
1954                         }
1955
1956                         vma->ops->clear_pages(vma);
1957
1958                         err = context_sync(ce);
1959                         if (err) {
1960                                 pr_err("%s: dummy setup timed out\n",
1961                                        ce->engine->name);
1962                                 goto end;
1963                         }
1964
1965                         vma = i915_vma_instance(act, vm, NULL);
1966                         if (IS_ERR(vma)) {
1967                                 err = PTR_ERR(vma);
1968                                 goto end;
1969                         }
1970
1971                         err = vma->ops->set_pages(vma);
1972                         if (err)
1973                                 goto end;
1974
1975                         /* Replace the TLB with target batches */
1976                         for (i = 0; i < count; i++) {
1977                                 struct i915_request *rq;
1978                                 u32 *cs = batch + i * 64 / sizeof(*cs);
1979                                 u64 addr;
1980
1981                                 vma->node.start = offset + i * PAGE_SIZE;
1982                                 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
1983
1984                                 addr = vma->node.start + i * 64;
1985                                 cs[4] = MI_NOOP;
1986                                 cs[6] = lower_32_bits(addr);
1987                                 cs[7] = upper_32_bits(addr);
1988                                 wmb();
1989
1990                                 rq = submit_batch(ce, addr);
1991                                 if (IS_ERR(rq)) {
1992                                         err = PTR_ERR(rq);
1993                                         goto end;
1994                                 }
1995
1996                                 /* Wait until the context chain has started */
1997                                 if (i == 0) {
1998                                         while (READ_ONCE(result[i]) &&
1999                                                !i915_request_completed(rq))
2000                                                 cond_resched();
2001                                 } else {
2002                                         end_spin(batch, i - 1);
2003                                 }
2004
2005                                 i915_request_put(rq);
2006                         }
2007                         end_spin(batch, count - 1);
2008
2009                         vma->ops->clear_pages(vma);
2010
2011                         err = context_sync(ce);
2012                         if (err) {
2013                                 pr_err("%s: writes timed out\n",
2014                                        ce->engine->name);
2015                                 goto end;
2016                         }
2017
2018                         for (i = 0; i < count; i++) {
2019                                 if (result[i] != i) {
2020                                         pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n",
2021                                                ce->engine->name, pass,
2022                                                offset, i, result[i], i);
2023                                         err = -EINVAL;
2024                                         goto end;
2025                                 }
2026                         }
2027
2028                         vm->clear_range(vm, offset, chunk_size);
2029                         pass++;
2030                 }
2031         }
2032 end:
2033         if (igt_flush_test(i915))
2034                 err = -EIO;
2035         i915_gem_context_unlock_engines(ctx);
2036         i915_gem_object_unpin_map(out);
2037 out_put_out:
2038         i915_gem_object_put(out);
2039 out_put_batch:
2040         i915_gem_object_unpin_map(act);
2041 out_put_act:
2042         i915_gem_object_put(act);
2043 out_put_bbe:
2044         i915_gem_object_put(bbe);
2045 out_vm:
2046         i915_vm_put(vm);
2047 out_unlock:
2048         fput(file);
2049         return err;
2050 }
2051
2052 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
2053 {
2054         static const struct i915_subtest tests[] = {
2055                 SUBTEST(igt_ppgtt_alloc),
2056                 SUBTEST(igt_ppgtt_lowlevel),
2057                 SUBTEST(igt_ppgtt_drunk),
2058                 SUBTEST(igt_ppgtt_walk),
2059                 SUBTEST(igt_ppgtt_pot),
2060                 SUBTEST(igt_ppgtt_fill),
2061                 SUBTEST(igt_ppgtt_shrink),
2062                 SUBTEST(igt_ppgtt_shrink_boom),
2063                 SUBTEST(igt_ggtt_lowlevel),
2064                 SUBTEST(igt_ggtt_drunk),
2065                 SUBTEST(igt_ggtt_walk),
2066                 SUBTEST(igt_ggtt_pot),
2067                 SUBTEST(igt_ggtt_fill),
2068                 SUBTEST(igt_ggtt_page),
2069                 SUBTEST(igt_cs_tlb),
2070         };
2071
2072         GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total));
2073
2074         return i915_subtests(tests, i915);
2075 }