2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include <linux/list_sort.h>
26 #include <linux/prime_numbers.h>
28 #include "gem/i915_gem_context.h"
29 #include "gem/selftests/mock_context.h"
30 #include "gt/intel_context.h"
32 #include "i915_random.h"
33 #include "i915_selftest.h"
36 #include "mock_gem_device.h"
38 #include "igt_flush_test.h"
40 static void cleanup_freed_objects(struct drm_i915_private *i915)
42 i915_gem_drain_freed_objects(i915);
45 static void fake_free_pages(struct drm_i915_gem_object *obj,
46 struct sg_table *pages)
52 static int fake_get_pages(struct drm_i915_gem_object *obj)
54 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
55 #define PFN_BIAS 0x1000
56 struct sg_table *pages;
57 struct scatterlist *sg;
58 unsigned int sg_page_sizes;
59 typeof(obj->base.size) rem;
61 pages = kmalloc(sizeof(*pages), GFP);
65 rem = round_up(obj->base.size, BIT(31)) >> 31;
66 if (sg_alloc_table(pages, rem, GFP)) {
73 for (sg = pages->sgl; sg; sg = sg_next(sg)) {
74 unsigned long len = min_t(typeof(rem), rem, BIT(31));
77 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
78 sg_dma_address(sg) = page_to_phys(sg_page(sg));
86 __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
92 static void fake_put_pages(struct drm_i915_gem_object *obj,
93 struct sg_table *pages)
95 fake_free_pages(obj, pages);
96 obj->mm.dirty = false;
99 static const struct drm_i915_gem_object_ops fake_ops = {
101 .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
102 .get_pages = fake_get_pages,
103 .put_pages = fake_put_pages,
106 static struct drm_i915_gem_object *
107 fake_dma_object(struct drm_i915_private *i915, u64 size)
109 static struct lock_class_key lock_class;
110 struct drm_i915_gem_object *obj;
113 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
115 if (overflows_type(size, obj->base.size))
116 return ERR_PTR(-E2BIG);
118 obj = i915_gem_object_alloc();
122 drm_gem_private_object_init(&i915->drm, &obj->base, size);
123 i915_gem_object_init(obj, &fake_ops, &lock_class);
125 i915_gem_object_set_volatile(obj);
127 obj->write_domain = I915_GEM_DOMAIN_CPU;
128 obj->read_domains = I915_GEM_DOMAIN_CPU;
129 obj->cache_level = I915_CACHE_NONE;
131 /* Preallocate the "backing storage" */
132 if (i915_gem_object_pin_pages(obj))
135 i915_gem_object_unpin_pages(obj);
139 i915_gem_object_put(obj);
141 return ERR_PTR(-ENOMEM);
144 static int igt_ppgtt_alloc(void *arg)
146 struct drm_i915_private *dev_priv = arg;
147 struct i915_ppgtt *ppgtt;
148 u64 size, last, limit;
151 /* Allocate a ppggt and try to fill the entire range */
153 if (!HAS_PPGTT(dev_priv))
156 ppgtt = i915_ppgtt_create(&dev_priv->gt);
158 return PTR_ERR(ppgtt);
160 if (!ppgtt->vm.allocate_va_range)
161 goto err_ppgtt_cleanup;
164 * While we only allocate the page tables here and so we could
165 * address a much larger GTT than we could actually fit into
166 * RAM, a practical limit is the amount of physical pages in the system.
167 * This should ensure that we do not run into the oomkiller during
168 * the test and take down the machine wilfully.
170 limit = totalram_pages() << PAGE_SHIFT;
171 limit = min(ppgtt->vm.total, limit);
173 /* Check we can allocate the entire range */
174 for (size = 4096; size <= limit; size <<= 2) {
175 struct i915_vm_pt_stash stash = {};
177 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size);
179 goto err_ppgtt_cleanup;
181 err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);
183 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
184 goto err_ppgtt_cleanup;
187 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size);
190 ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
192 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
195 /* Check we can incrementally allocate the entire range */
196 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
197 struct i915_vm_pt_stash stash = {};
199 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last);
201 goto err_ppgtt_cleanup;
203 err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);
205 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
206 goto err_ppgtt_cleanup;
209 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash,
213 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
217 i915_vm_put(&ppgtt->vm);
221 static int lowlevel_hole(struct i915_address_space *vm,
222 u64 hole_start, u64 hole_end,
223 unsigned long end_time)
225 I915_RND_STATE(seed_prng);
226 struct i915_vma *mock_vma;
229 mock_vma = kzalloc(sizeof(*mock_vma), GFP_KERNEL);
233 /* Keep creating larger objects until one cannot fit into the hole */
234 for (size = 12; (hole_end - hole_start) >> size; size++) {
235 I915_RND_SUBSTATE(prng, seed_prng);
236 struct drm_i915_gem_object *obj;
237 unsigned int *order, count, n;
240 hole_size = (hole_end - hole_start) >> size;
241 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
242 hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
243 count = hole_size >> 1;
245 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
246 __func__, hole_start, hole_end, size, hole_size);
251 order = i915_random_order(count, &prng);
254 } while (count >>= 1);
261 GEM_BUG_ON(count * BIT_ULL(size) > vm->total);
262 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end);
264 /* Ignore allocation failures (i.e. don't report them as
265 * a test failure) as we are purposefully allocating very
266 * large objects without checking that we have sufficient
267 * memory. We expect to hit -ENOMEM.
270 obj = fake_dma_object(vm->i915, BIT_ULL(size));
276 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
278 if (i915_gem_object_pin_pages(obj)) {
279 i915_gem_object_put(obj);
284 for (n = 0; n < count; n++) {
285 u64 addr = hole_start + order[n] * BIT_ULL(size);
286 intel_wakeref_t wakeref;
288 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
290 if (igt_timeout(end_time,
291 "%s timed out before %d/%d\n",
292 __func__, n, count)) {
293 hole_end = hole_start; /* quit */
297 if (vm->allocate_va_range) {
298 struct i915_vm_pt_stash stash = {};
300 if (i915_vm_alloc_pt_stash(vm, &stash,
304 if (i915_vm_pin_pt_stash(vm, &stash)) {
305 i915_vm_free_pt_stash(vm, &stash);
309 vm->allocate_va_range(vm, &stash,
310 addr, BIT_ULL(size));
312 i915_vm_free_pt_stash(vm, &stash);
315 mock_vma->pages = obj->mm.pages;
316 mock_vma->node.size = BIT_ULL(size);
317 mock_vma->node.start = addr;
319 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
320 vm->insert_entries(vm, mock_vma,
325 i915_random_reorder(order, count, &prng);
326 for (n = 0; n < count; n++) {
327 u64 addr = hole_start + order[n] * BIT_ULL(size);
328 intel_wakeref_t wakeref;
330 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
331 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
332 vm->clear_range(vm, addr, BIT_ULL(size));
335 i915_gem_object_unpin_pages(obj);
336 i915_gem_object_put(obj);
340 cleanup_freed_objects(vm->i915);
347 static void close_object_list(struct list_head *objects,
348 struct i915_address_space *vm)
350 struct drm_i915_gem_object *obj, *on;
353 list_for_each_entry_safe(obj, on, objects, st_link) {
354 struct i915_vma *vma;
356 vma = i915_vma_instance(obj, vm, NULL);
358 ignored = i915_vma_unbind(vma);
360 list_del(&obj->st_link);
361 i915_gem_object_put(obj);
365 static int fill_hole(struct i915_address_space *vm,
366 u64 hole_start, u64 hole_end,
367 unsigned long end_time)
369 const u64 hole_size = hole_end - hole_start;
370 struct drm_i915_gem_object *obj;
371 const unsigned long max_pages =
372 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT);
373 const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
374 unsigned long npages, prime, flags;
375 struct i915_vma *vma;
379 /* Try binding many VMA working inwards from either edge */
381 flags = PIN_OFFSET_FIXED | PIN_USER;
382 if (i915_is_ggtt(vm))
385 for_each_prime_number_from(prime, 2, max_step) {
386 for (npages = 1; npages <= max_pages; npages *= prime) {
387 const u64 full_size = npages << PAGE_SHIFT;
393 { "top-down", hole_end, -1, },
394 { "bottom-up", hole_start, 1, },
398 obj = fake_dma_object(vm->i915, full_size);
402 list_add(&obj->st_link, &objects);
404 /* Align differing sized objects against the edges, and
405 * check we don't walk off into the void when binding
408 for (p = phases; p->name; p++) {
412 list_for_each_entry(obj, &objects, st_link) {
413 vma = i915_vma_instance(obj, vm, NULL);
418 if (offset < hole_start + obj->base.size)
420 offset -= obj->base.size;
423 err = i915_vma_pin(vma, 0, 0, offset | flags);
425 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
426 __func__, p->name, err, npages, prime, offset);
430 if (!drm_mm_node_allocated(&vma->node) ||
431 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
432 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
433 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
442 if (offset + obj->base.size > hole_end)
444 offset += obj->base.size;
449 list_for_each_entry(obj, &objects, st_link) {
450 vma = i915_vma_instance(obj, vm, NULL);
455 if (offset < hole_start + obj->base.size)
457 offset -= obj->base.size;
460 if (!drm_mm_node_allocated(&vma->node) ||
461 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
462 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
463 __func__, p->name, vma->node.start, vma->node.size,
469 err = i915_vma_unbind(vma);
471 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
472 __func__, p->name, vma->node.start, vma->node.size,
478 if (offset + obj->base.size > hole_end)
480 offset += obj->base.size;
485 list_for_each_entry_reverse(obj, &objects, st_link) {
486 vma = i915_vma_instance(obj, vm, NULL);
491 if (offset < hole_start + obj->base.size)
493 offset -= obj->base.size;
496 err = i915_vma_pin(vma, 0, 0, offset | flags);
498 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
499 __func__, p->name, err, npages, prime, offset);
503 if (!drm_mm_node_allocated(&vma->node) ||
504 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
505 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
506 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
515 if (offset + obj->base.size > hole_end)
517 offset += obj->base.size;
522 list_for_each_entry_reverse(obj, &objects, st_link) {
523 vma = i915_vma_instance(obj, vm, NULL);
528 if (offset < hole_start + obj->base.size)
530 offset -= obj->base.size;
533 if (!drm_mm_node_allocated(&vma->node) ||
534 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
535 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
536 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
542 err = i915_vma_unbind(vma);
544 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
545 __func__, p->name, vma->node.start, vma->node.size,
551 if (offset + obj->base.size > hole_end)
553 offset += obj->base.size;
558 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n",
559 __func__, npages, prime)) {
565 close_object_list(&objects, vm);
566 cleanup_freed_objects(vm->i915);
572 close_object_list(&objects, vm);
576 static int walk_hole(struct i915_address_space *vm,
577 u64 hole_start, u64 hole_end,
578 unsigned long end_time)
580 const u64 hole_size = hole_end - hole_start;
581 const unsigned long max_pages =
582 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
586 /* Try binding a single VMA in different positions within the hole */
588 flags = PIN_OFFSET_FIXED | PIN_USER;
589 if (i915_is_ggtt(vm))
592 for_each_prime_number_from(size, 1, max_pages) {
593 struct drm_i915_gem_object *obj;
594 struct i915_vma *vma;
598 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT);
602 vma = i915_vma_instance(obj, vm, NULL);
608 for (addr = hole_start;
609 addr + obj->base.size < hole_end;
610 addr += obj->base.size) {
611 err = i915_vma_pin(vma, 0, 0, addr | flags);
613 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
614 __func__, addr, vma->size,
615 hole_start, hole_end, err);
620 if (!drm_mm_node_allocated(&vma->node) ||
621 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
622 pr_err("%s incorrect at %llx + %llx\n",
623 __func__, addr, vma->size);
628 err = i915_vma_unbind(vma);
630 pr_err("%s unbind failed at %llx + %llx with err=%d\n",
631 __func__, addr, vma->size, err);
635 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
637 if (igt_timeout(end_time,
638 "%s timed out at %llx\n",
646 i915_gem_object_put(obj);
650 cleanup_freed_objects(vm->i915);
656 static int pot_hole(struct i915_address_space *vm,
657 u64 hole_start, u64 hole_end,
658 unsigned long end_time)
660 struct drm_i915_gem_object *obj;
661 struct i915_vma *vma;
666 flags = PIN_OFFSET_FIXED | PIN_USER;
667 if (i915_is_ggtt(vm))
670 obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE);
674 vma = i915_vma_instance(obj, vm, NULL);
680 /* Insert a pair of pages across every pot boundary within the hole */
681 for (pot = fls64(hole_end - 1) - 1;
682 pot > ilog2(2 * I915_GTT_PAGE_SIZE);
684 u64 step = BIT_ULL(pot);
687 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
688 addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
690 err = i915_vma_pin(vma, 0, 0, addr | flags);
692 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
695 hole_start, hole_end,
700 if (!drm_mm_node_allocated(&vma->node) ||
701 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
702 pr_err("%s incorrect at %llx + %llx\n",
703 __func__, addr, vma->size);
705 err = i915_vma_unbind(vma);
711 err = i915_vma_unbind(vma);
715 if (igt_timeout(end_time,
716 "%s timed out after %d/%d\n",
717 __func__, pot, fls64(hole_end - 1) - 1)) {
724 i915_gem_object_put(obj);
728 static int drunk_hole(struct i915_address_space *vm,
729 u64 hole_start, u64 hole_end,
730 unsigned long end_time)
732 I915_RND_STATE(prng);
736 flags = PIN_OFFSET_FIXED | PIN_USER;
737 if (i915_is_ggtt(vm))
740 /* Keep creating larger objects until one cannot fit into the hole */
741 for (size = 12; (hole_end - hole_start) >> size; size++) {
742 struct drm_i915_gem_object *obj;
743 unsigned int *order, count, n;
744 struct i915_vma *vma;
748 hole_size = (hole_end - hole_start) >> size;
749 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
750 hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
751 count = hole_size >> 1;
753 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
754 __func__, hole_start, hole_end, size, hole_size);
759 order = i915_random_order(count, &prng);
762 } while (count >>= 1);
767 /* Ignore allocation failures (i.e. don't report them as
768 * a test failure) as we are purposefully allocating very
769 * large objects without checking that we have sufficient
770 * memory. We expect to hit -ENOMEM.
773 obj = fake_dma_object(vm->i915, BIT_ULL(size));
779 vma = i915_vma_instance(obj, vm, NULL);
785 GEM_BUG_ON(vma->size != BIT_ULL(size));
787 for (n = 0; n < count; n++) {
788 u64 addr = hole_start + order[n] * BIT_ULL(size);
790 err = i915_vma_pin(vma, 0, 0, addr | flags);
792 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
795 hole_start, hole_end,
800 if (!drm_mm_node_allocated(&vma->node) ||
801 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
802 pr_err("%s incorrect at %llx + %llx\n",
803 __func__, addr, BIT_ULL(size));
805 err = i915_vma_unbind(vma);
811 err = i915_vma_unbind(vma);
814 if (igt_timeout(end_time,
815 "%s timed out after %d/%d\n",
816 __func__, n, count)) {
823 i915_gem_object_put(obj);
828 cleanup_freed_objects(vm->i915);
834 static int __shrink_hole(struct i915_address_space *vm,
835 u64 hole_start, u64 hole_end,
836 unsigned long end_time)
838 struct drm_i915_gem_object *obj;
839 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
840 unsigned int order = 12;
845 /* Keep creating larger objects until one cannot fit into the hole */
846 for (addr = hole_start; addr < hole_end; ) {
847 struct i915_vma *vma;
848 u64 size = BIT_ULL(order++);
850 size = min(size, hole_end - addr);
851 obj = fake_dma_object(vm->i915, size);
857 list_add(&obj->st_link, &objects);
859 vma = i915_vma_instance(obj, vm, NULL);
865 GEM_BUG_ON(vma->size != size);
867 err = i915_vma_pin(vma, 0, 0, addr | flags);
869 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
870 __func__, addr, size, hole_start, hole_end, err);
874 if (!drm_mm_node_allocated(&vma->node) ||
875 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
876 pr_err("%s incorrect at %llx + %llx\n",
877 __func__, addr, size);
879 err = i915_vma_unbind(vma);
888 * Since we are injecting allocation faults at random intervals,
889 * wait for this allocation to complete before we change the
892 err = i915_vma_sync(vma);
896 if (igt_timeout(end_time,
897 "%s timed out at ofset %llx [%llx - %llx]\n",
898 __func__, addr, hole_start, hole_end)) {
904 close_object_list(&objects, vm);
905 cleanup_freed_objects(vm->i915);
909 static int shrink_hole(struct i915_address_space *vm,
910 u64 hole_start, u64 hole_end,
911 unsigned long end_time)
916 vm->fault_attr.probability = 999;
917 atomic_set(&vm->fault_attr.times, -1);
919 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
920 vm->fault_attr.interval = prime;
921 err = __shrink_hole(vm, hole_start, hole_end, end_time);
926 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
931 static int shrink_boom(struct i915_address_space *vm,
932 u64 hole_start, u64 hole_end,
933 unsigned long end_time)
935 unsigned int sizes[] = { SZ_2M, SZ_1G };
936 struct drm_i915_gem_object *purge;
937 struct drm_i915_gem_object *explode;
942 * Catch the case which shrink_hole seems to miss. The setup here
943 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while
944 * ensuring that all vma assiocated with the respective pd/pdp are
945 * unpinned at the time.
948 for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
949 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
950 unsigned int size = sizes[i];
951 struct i915_vma *vma;
953 purge = fake_dma_object(vm->i915, size);
955 return PTR_ERR(purge);
957 vma = i915_vma_instance(purge, vm, NULL);
963 err = i915_vma_pin(vma, 0, 0, flags);
967 /* Should now be ripe for purging */
970 explode = fake_dma_object(vm->i915, size);
971 if (IS_ERR(explode)) {
972 err = PTR_ERR(explode);
976 vm->fault_attr.probability = 100;
977 vm->fault_attr.interval = 1;
978 atomic_set(&vm->fault_attr.times, -1);
980 vma = i915_vma_instance(explode, vm, NULL);
986 err = i915_vma_pin(vma, 0, 0, flags | size);
992 i915_gem_object_put(purge);
993 i915_gem_object_put(explode);
995 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
996 cleanup_freed_objects(vm->i915);
1002 i915_gem_object_put(explode);
1004 i915_gem_object_put(purge);
1005 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1009 static int exercise_ppgtt(struct drm_i915_private *dev_priv,
1010 int (*func)(struct i915_address_space *vm,
1011 u64 hole_start, u64 hole_end,
1012 unsigned long end_time))
1014 struct i915_ppgtt *ppgtt;
1015 IGT_TIMEOUT(end_time);
1019 if (!HAS_FULL_PPGTT(dev_priv))
1022 file = mock_file(dev_priv);
1024 return PTR_ERR(file);
1026 ppgtt = i915_ppgtt_create(&dev_priv->gt);
1027 if (IS_ERR(ppgtt)) {
1028 err = PTR_ERR(ppgtt);
1031 GEM_BUG_ON(offset_in_page(ppgtt->vm.total));
1032 GEM_BUG_ON(!atomic_read(&ppgtt->vm.open));
1034 err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time);
1036 i915_vm_put(&ppgtt->vm);
1043 static int igt_ppgtt_fill(void *arg)
1045 return exercise_ppgtt(arg, fill_hole);
1048 static int igt_ppgtt_walk(void *arg)
1050 return exercise_ppgtt(arg, walk_hole);
1053 static int igt_ppgtt_pot(void *arg)
1055 return exercise_ppgtt(arg, pot_hole);
1058 static int igt_ppgtt_drunk(void *arg)
1060 return exercise_ppgtt(arg, drunk_hole);
1063 static int igt_ppgtt_lowlevel(void *arg)
1065 return exercise_ppgtt(arg, lowlevel_hole);
1068 static int igt_ppgtt_shrink(void *arg)
1070 return exercise_ppgtt(arg, shrink_hole);
1073 static int igt_ppgtt_shrink_boom(void *arg)
1075 return exercise_ppgtt(arg, shrink_boom);
1078 static int sort_holes(void *priv, const struct list_head *A,
1079 const struct list_head *B)
1081 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
1082 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);
1084 if (a->start < b->start)
1090 static int exercise_ggtt(struct drm_i915_private *i915,
1091 int (*func)(struct i915_address_space *vm,
1092 u64 hole_start, u64 hole_end,
1093 unsigned long end_time))
1095 struct i915_ggtt *ggtt = &i915->ggtt;
1096 u64 hole_start, hole_end, last = 0;
1097 struct drm_mm_node *node;
1098 IGT_TIMEOUT(end_time);
1102 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes);
1103 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) {
1104 if (hole_start < last)
1107 if (ggtt->vm.mm.color_adjust)
1108 ggtt->vm.mm.color_adjust(node, 0,
1109 &hole_start, &hole_end);
1110 if (hole_start >= hole_end)
1113 err = func(&ggtt->vm, hole_start, hole_end, end_time);
1117 /* As we have manipulated the drm_mm, the list may be corrupt */
1125 static int igt_ggtt_fill(void *arg)
1127 return exercise_ggtt(arg, fill_hole);
1130 static int igt_ggtt_walk(void *arg)
1132 return exercise_ggtt(arg, walk_hole);
1135 static int igt_ggtt_pot(void *arg)
1137 return exercise_ggtt(arg, pot_hole);
1140 static int igt_ggtt_drunk(void *arg)
1142 return exercise_ggtt(arg, drunk_hole);
1145 static int igt_ggtt_lowlevel(void *arg)
1147 return exercise_ggtt(arg, lowlevel_hole);
1150 static int igt_ggtt_page(void *arg)
1152 const unsigned int count = PAGE_SIZE/sizeof(u32);
1153 I915_RND_STATE(prng);
1154 struct drm_i915_private *i915 = arg;
1155 struct i915_ggtt *ggtt = &i915->ggtt;
1156 struct drm_i915_gem_object *obj;
1157 intel_wakeref_t wakeref;
1158 struct drm_mm_node tmp;
1159 unsigned int *order, n;
1162 if (!i915_ggtt_has_aperture(ggtt))
1165 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1167 return PTR_ERR(obj);
1169 err = i915_gem_object_pin_pages(obj);
1173 memset(&tmp, 0, sizeof(tmp));
1174 mutex_lock(&ggtt->vm.mutex);
1175 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp,
1176 count * PAGE_SIZE, 0,
1177 I915_COLOR_UNEVICTABLE,
1178 0, ggtt->mappable_end,
1180 mutex_unlock(&ggtt->vm.mutex);
1184 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1186 for (n = 0; n < count; n++) {
1187 u64 offset = tmp.start + n * PAGE_SIZE;
1189 ggtt->vm.insert_page(&ggtt->vm,
1190 i915_gem_object_get_dma_address(obj, 0),
1191 offset, I915_CACHE_NONE, 0);
1194 order = i915_random_order(count, &prng);
1200 for (n = 0; n < count; n++) {
1201 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1204 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1205 iowrite32(n, vaddr + n);
1206 io_mapping_unmap_atomic(vaddr);
1208 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1210 i915_random_reorder(order, count, &prng);
1211 for (n = 0; n < count; n++) {
1212 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1216 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1217 val = ioread32(vaddr + n);
1218 io_mapping_unmap_atomic(vaddr);
1221 pr_err("insert page failed: found %d, expected %d\n",
1230 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size);
1231 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1232 mutex_lock(&ggtt->vm.mutex);
1233 drm_mm_remove_node(&tmp);
1234 mutex_unlock(&ggtt->vm.mutex);
1236 i915_gem_object_unpin_pages(obj);
1238 i915_gem_object_put(obj);
1242 static void track_vma_bind(struct i915_vma *vma)
1244 struct drm_i915_gem_object *obj = vma->obj;
1246 __i915_gem_object_pin_pages(obj);
1248 GEM_BUG_ON(vma->pages);
1249 atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE);
1250 __i915_gem_object_pin_pages(obj);
1251 vma->pages = obj->mm.pages;
1253 mutex_lock(&vma->vm->mutex);
1254 list_add_tail(&vma->vm_link, &vma->vm->bound_list);
1255 mutex_unlock(&vma->vm->mutex);
1258 static int exercise_mock(struct drm_i915_private *i915,
1259 int (*func)(struct i915_address_space *vm,
1260 u64 hole_start, u64 hole_end,
1261 unsigned long end_time))
1263 const u64 limit = totalram_pages() << PAGE_SHIFT;
1264 struct i915_address_space *vm;
1265 struct i915_gem_context *ctx;
1266 IGT_TIMEOUT(end_time);
1269 ctx = mock_context(i915, "mock");
1273 vm = i915_gem_context_get_vm_rcu(ctx);
1274 err = func(vm, 0, min(vm->total, limit), end_time);
1277 mock_context_close(ctx);
1281 static int igt_mock_fill(void *arg)
1283 struct i915_ggtt *ggtt = arg;
1285 return exercise_mock(ggtt->vm.i915, fill_hole);
1288 static int igt_mock_walk(void *arg)
1290 struct i915_ggtt *ggtt = arg;
1292 return exercise_mock(ggtt->vm.i915, walk_hole);
1295 static int igt_mock_pot(void *arg)
1297 struct i915_ggtt *ggtt = arg;
1299 return exercise_mock(ggtt->vm.i915, pot_hole);
1302 static int igt_mock_drunk(void *arg)
1304 struct i915_ggtt *ggtt = arg;
1306 return exercise_mock(ggtt->vm.i915, drunk_hole);
1309 static int igt_gtt_reserve(void *arg)
1311 struct i915_ggtt *ggtt = arg;
1312 struct drm_i915_gem_object *obj, *on;
1313 I915_RND_STATE(prng);
1318 /* i915_gem_gtt_reserve() tries to reserve the precise range
1319 * for the node, and evicts if it has to. So our test checks that
1320 * it can give us the requsted space and prevent overlaps.
1323 /* Start by filling the GGTT */
1325 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1326 total += 2 * I915_GTT_PAGE_SIZE) {
1327 struct i915_vma *vma;
1329 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1336 err = i915_gem_object_pin_pages(obj);
1338 i915_gem_object_put(obj);
1342 list_add(&obj->st_link, &objects);
1344 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1350 mutex_lock(&ggtt->vm.mutex);
1351 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1356 mutex_unlock(&ggtt->vm.mutex);
1358 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1359 total, ggtt->vm.total, err);
1362 track_vma_bind(vma);
1364 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1365 if (vma->node.start != total ||
1366 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1367 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1368 vma->node.start, vma->node.size,
1369 total, 2*I915_GTT_PAGE_SIZE);
1375 /* Now we start forcing evictions */
1376 for (total = I915_GTT_PAGE_SIZE;
1377 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1378 total += 2 * I915_GTT_PAGE_SIZE) {
1379 struct i915_vma *vma;
1381 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1388 err = i915_gem_object_pin_pages(obj);
1390 i915_gem_object_put(obj);
1394 list_add(&obj->st_link, &objects);
1396 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1402 mutex_lock(&ggtt->vm.mutex);
1403 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1408 mutex_unlock(&ggtt->vm.mutex);
1410 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1411 total, ggtt->vm.total, err);
1414 track_vma_bind(vma);
1416 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1417 if (vma->node.start != total ||
1418 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1419 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1420 vma->node.start, vma->node.size,
1421 total, 2*I915_GTT_PAGE_SIZE);
1427 /* And then try at random */
1428 list_for_each_entry_safe(obj, on, &objects, st_link) {
1429 struct i915_vma *vma;
1432 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1438 err = i915_vma_unbind(vma);
1440 pr_err("i915_vma_unbind failed with err=%d!\n", err);
1444 offset = igt_random_offset(&prng,
1446 2 * I915_GTT_PAGE_SIZE,
1447 I915_GTT_MIN_ALIGNMENT);
1449 mutex_lock(&ggtt->vm.mutex);
1450 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1455 mutex_unlock(&ggtt->vm.mutex);
1457 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1458 total, ggtt->vm.total, err);
1461 track_vma_bind(vma);
1463 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1464 if (vma->node.start != offset ||
1465 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1466 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1467 vma->node.start, vma->node.size,
1468 offset, 2*I915_GTT_PAGE_SIZE);
1475 list_for_each_entry_safe(obj, on, &objects, st_link) {
1476 i915_gem_object_unpin_pages(obj);
1477 i915_gem_object_put(obj);
1482 static int igt_gtt_insert(void *arg)
1484 struct i915_ggtt *ggtt = arg;
1485 struct drm_i915_gem_object *obj, *on;
1486 struct drm_mm_node tmp = {};
1487 const struct invalid_insert {
1491 } invalid_insert[] = {
1493 ggtt->vm.total + I915_GTT_PAGE_SIZE, 0,
1497 2*I915_GTT_PAGE_SIZE, 0,
1498 0, I915_GTT_PAGE_SIZE,
1501 -(u64)I915_GTT_PAGE_SIZE, 0,
1502 0, 4*I915_GTT_PAGE_SIZE,
1505 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE,
1506 0, 4*I915_GTT_PAGE_SIZE,
1509 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1,
1510 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1,
1518 /* i915_gem_gtt_insert() tries to allocate some free space in the GTT
1519 * to the node, evicting if required.
1522 /* Check a couple of obviously invalid requests */
1523 for (ii = invalid_insert; ii->size; ii++) {
1524 mutex_lock(&ggtt->vm.mutex);
1525 err = i915_gem_gtt_insert(&ggtt->vm, &tmp,
1526 ii->size, ii->alignment,
1527 I915_COLOR_UNEVICTABLE,
1530 mutex_unlock(&ggtt->vm.mutex);
1531 if (err != -ENOSPC) {
1532 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1533 ii->size, ii->alignment, ii->start, ii->end,
1539 /* Start by filling the GGTT */
1541 total + I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1542 total += I915_GTT_PAGE_SIZE) {
1543 struct i915_vma *vma;
1545 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1546 I915_GTT_PAGE_SIZE);
1552 err = i915_gem_object_pin_pages(obj);
1554 i915_gem_object_put(obj);
1558 list_add(&obj->st_link, &objects);
1560 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1566 mutex_lock(&ggtt->vm.mutex);
1567 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1568 obj->base.size, 0, obj->cache_level,
1571 mutex_unlock(&ggtt->vm.mutex);
1572 if (err == -ENOSPC) {
1573 /* maxed out the GGTT space */
1574 i915_gem_object_put(obj);
1578 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1579 total, ggtt->vm.total, err);
1582 track_vma_bind(vma);
1583 __i915_vma_pin(vma);
1585 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1588 list_for_each_entry(obj, &objects, st_link) {
1589 struct i915_vma *vma;
1591 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1597 if (!drm_mm_node_allocated(&vma->node)) {
1598 pr_err("VMA was unexpectedly evicted!\n");
1603 __i915_vma_unpin(vma);
1606 /* If we then reinsert, we should find the same hole */
1607 list_for_each_entry_safe(obj, on, &objects, st_link) {
1608 struct i915_vma *vma;
1611 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1617 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1618 offset = vma->node.start;
1620 err = i915_vma_unbind(vma);
1622 pr_err("i915_vma_unbind failed with err=%d!\n", err);
1626 mutex_lock(&ggtt->vm.mutex);
1627 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1628 obj->base.size, 0, obj->cache_level,
1631 mutex_unlock(&ggtt->vm.mutex);
1633 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1634 total, ggtt->vm.total, err);
1637 track_vma_bind(vma);
1639 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1640 if (vma->node.start != offset) {
1641 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n",
1642 offset, vma->node.start);
1648 /* And then force evictions */
1650 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1651 total += 2 * I915_GTT_PAGE_SIZE) {
1652 struct i915_vma *vma;
1654 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1655 2 * I915_GTT_PAGE_SIZE);
1661 err = i915_gem_object_pin_pages(obj);
1663 i915_gem_object_put(obj);
1667 list_add(&obj->st_link, &objects);
1669 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1675 mutex_lock(&ggtt->vm.mutex);
1676 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1677 obj->base.size, 0, obj->cache_level,
1680 mutex_unlock(&ggtt->vm.mutex);
1682 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1683 total, ggtt->vm.total, err);
1686 track_vma_bind(vma);
1688 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1692 list_for_each_entry_safe(obj, on, &objects, st_link) {
1693 i915_gem_object_unpin_pages(obj);
1694 i915_gem_object_put(obj);
1699 int i915_gem_gtt_mock_selftests(void)
1701 static const struct i915_subtest tests[] = {
1702 SUBTEST(igt_mock_drunk),
1703 SUBTEST(igt_mock_walk),
1704 SUBTEST(igt_mock_pot),
1705 SUBTEST(igt_mock_fill),
1706 SUBTEST(igt_gtt_reserve),
1707 SUBTEST(igt_gtt_insert),
1709 struct drm_i915_private *i915;
1710 struct i915_ggtt *ggtt;
1713 i915 = mock_gem_device();
1717 ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL);
1722 mock_init_ggtt(i915, ggtt);
1724 err = i915_subtests(tests, ggtt);
1726 mock_device_flush(i915);
1727 i915_gem_drain_freed_objects(i915);
1728 mock_fini_ggtt(ggtt);
1731 mock_destroy_device(i915);
1735 static int context_sync(struct intel_context *ce)
1737 struct i915_request *rq;
1740 rq = intel_context_create_request(ce);
1744 i915_request_get(rq);
1745 i915_request_add(rq);
1747 timeout = i915_request_wait(rq, 0, HZ / 5);
1748 i915_request_put(rq);
1750 return timeout < 0 ? -EIO : 0;
1753 static struct i915_request *
1754 submit_batch(struct intel_context *ce, u64 addr)
1756 struct i915_request *rq;
1759 rq = intel_context_create_request(ce);
1764 if (rq->engine->emit_init_breadcrumb) /* detect a hang */
1765 err = rq->engine->emit_init_breadcrumb(rq);
1767 err = rq->engine->emit_bb_start(rq, addr, 0, 0);
1770 i915_request_get(rq);
1771 i915_request_add(rq);
1773 return err ? ERR_PTR(err) : rq;
1776 static u32 *spinner(u32 *batch, int i)
1778 return batch + i * 64 / sizeof(*batch) + 4;
1781 static void end_spin(u32 *batch, int i)
1783 *spinner(batch, i) = MI_BATCH_BUFFER_END;
1787 static int igt_cs_tlb(void *arg)
1789 const unsigned int count = PAGE_SIZE / 64;
1790 const unsigned int chunk_size = count * PAGE_SIZE;
1791 struct drm_i915_private *i915 = arg;
1792 struct drm_i915_gem_object *bbe, *act, *out;
1793 struct i915_gem_engines_iter it;
1794 struct i915_address_space *vm;
1795 struct i915_gem_context *ctx;
1796 struct intel_context *ce;
1797 struct i915_vma *vma;
1798 I915_RND_STATE(prng);
1806 * Our mission here is to fool the hardware to execute something
1807 * from scratch as it has not seen the batch move (due to missing
1808 * the TLB invalidate).
1811 file = mock_file(i915);
1813 return PTR_ERR(file);
1815 ctx = live_context(i915, file);
1821 vm = i915_gem_context_get_vm_rcu(ctx);
1822 if (i915_is_ggtt(vm))
1825 /* Create two pages; dummy we prefill the TLB, and intended */
1826 bbe = i915_gem_object_create_internal(i915, PAGE_SIZE);
1832 batch = i915_gem_object_pin_map(bbe, I915_MAP_WC);
1833 if (IS_ERR(batch)) {
1834 err = PTR_ERR(batch);
1837 memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32));
1838 i915_gem_object_flush_map(bbe);
1839 i915_gem_object_unpin_map(bbe);
1841 act = i915_gem_object_create_internal(i915, PAGE_SIZE);
1847 /* Track the execution of each request by writing into different slot */
1848 batch = i915_gem_object_pin_map(act, I915_MAP_WC);
1849 if (IS_ERR(batch)) {
1850 err = PTR_ERR(batch);
1853 for (i = 0; i < count; i++) {
1854 u32 *cs = batch + i * 64 / sizeof(*cs);
1855 u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32);
1857 GEM_BUG_ON(INTEL_GEN(i915) < 6);
1858 cs[0] = MI_STORE_DWORD_IMM_GEN4;
1859 if (INTEL_GEN(i915) >= 8) {
1860 cs[1] = lower_32_bits(addr);
1861 cs[2] = upper_32_bits(addr);
1864 cs[5] = MI_BATCH_BUFFER_START_GEN8;
1867 cs[2] = lower_32_bits(addr);
1870 cs[5] = MI_BATCH_BUFFER_START;
1874 out = i915_gem_object_create_internal(i915, PAGE_SIZE);
1879 i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED);
1881 vma = i915_vma_instance(out, vm, NULL);
1887 err = i915_vma_pin(vma, 0, 0,
1890 (vm->total - PAGE_SIZE));
1893 GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE);
1895 result = i915_gem_object_pin_map(out, I915_MAP_WB);
1896 if (IS_ERR(result)) {
1897 err = PTR_ERR(result);
1901 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1902 IGT_TIMEOUT(end_time);
1903 unsigned long pass = 0;
1905 if (!intel_engine_can_store_dword(ce->engine))
1908 while (!__igt_timeout(end_time, NULL)) {
1909 struct i915_vm_pt_stash stash = {};
1910 struct i915_request *rq;
1913 offset = igt_random_offset(&prng,
1914 0, vm->total - PAGE_SIZE,
1915 chunk_size, PAGE_SIZE);
1917 memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32));
1919 vma = i915_vma_instance(bbe, vm, NULL);
1925 err = vma->ops->set_pages(vma);
1929 err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size);
1933 err = i915_vm_pin_pt_stash(vm, &stash);
1935 i915_vm_free_pt_stash(vm, &stash);
1939 vm->allocate_va_range(vm, &stash, offset, chunk_size);
1941 i915_vm_free_pt_stash(vm, &stash);
1943 /* Prime the TLB with the dummy pages */
1944 for (i = 0; i < count; i++) {
1945 vma->node.start = offset + i * PAGE_SIZE;
1946 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
1948 rq = submit_batch(ce, vma->node.start);
1953 i915_request_put(rq);
1956 vma->ops->clear_pages(vma);
1958 err = context_sync(ce);
1960 pr_err("%s: dummy setup timed out\n",
1965 vma = i915_vma_instance(act, vm, NULL);
1971 err = vma->ops->set_pages(vma);
1975 /* Replace the TLB with target batches */
1976 for (i = 0; i < count; i++) {
1977 struct i915_request *rq;
1978 u32 *cs = batch + i * 64 / sizeof(*cs);
1981 vma->node.start = offset + i * PAGE_SIZE;
1982 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
1984 addr = vma->node.start + i * 64;
1986 cs[6] = lower_32_bits(addr);
1987 cs[7] = upper_32_bits(addr);
1990 rq = submit_batch(ce, addr);
1996 /* Wait until the context chain has started */
1998 while (READ_ONCE(result[i]) &&
1999 !i915_request_completed(rq))
2002 end_spin(batch, i - 1);
2005 i915_request_put(rq);
2007 end_spin(batch, count - 1);
2009 vma->ops->clear_pages(vma);
2011 err = context_sync(ce);
2013 pr_err("%s: writes timed out\n",
2018 for (i = 0; i < count; i++) {
2019 if (result[i] != i) {
2020 pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n",
2021 ce->engine->name, pass,
2022 offset, i, result[i], i);
2028 vm->clear_range(vm, offset, chunk_size);
2033 if (igt_flush_test(i915))
2035 i915_gem_context_unlock_engines(ctx);
2036 i915_gem_object_unpin_map(out);
2038 i915_gem_object_put(out);
2040 i915_gem_object_unpin_map(act);
2042 i915_gem_object_put(act);
2044 i915_gem_object_put(bbe);
2052 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
2054 static const struct i915_subtest tests[] = {
2055 SUBTEST(igt_ppgtt_alloc),
2056 SUBTEST(igt_ppgtt_lowlevel),
2057 SUBTEST(igt_ppgtt_drunk),
2058 SUBTEST(igt_ppgtt_walk),
2059 SUBTEST(igt_ppgtt_pot),
2060 SUBTEST(igt_ppgtt_fill),
2061 SUBTEST(igt_ppgtt_shrink),
2062 SUBTEST(igt_ppgtt_shrink_boom),
2063 SUBTEST(igt_ggtt_lowlevel),
2064 SUBTEST(igt_ggtt_drunk),
2065 SUBTEST(igt_ggtt_walk),
2066 SUBTEST(igt_ggtt_pot),
2067 SUBTEST(igt_ggtt_fill),
2068 SUBTEST(igt_ggtt_page),
2069 SUBTEST(igt_cs_tlb),
2072 GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total));
2074 return i915_subtests(tests, i915);