2 * SPDX-License-Identifier: MIT
4 * Copyright © 2014-2016 Intel Corporation
8 #include "i915_gem_object.h"
9 #include "i915_scatterlist.h"
10 #include "i915_gem_lmem.h"
11 #include "i915_gem_mman.h"
13 #include "gt/intel_gt.h"
15 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
16 struct sg_table *pages,
17 unsigned int sg_page_sizes)
19 struct drm_i915_private *i915 = to_i915(obj->base.dev);
20 unsigned long supported = INTEL_INFO(i915)->page_sizes;
23 lockdep_assert_held(&obj->mm.lock);
25 if (i915_gem_object_is_volatile(obj))
26 obj->mm.madv = I915_MADV_DONTNEED;
28 /* Make the pages coherent with the GPU (flushing any swapin). */
29 if (obj->cache_dirty) {
30 obj->write_domain = 0;
31 if (i915_gem_object_has_struct_page(obj))
32 drm_clflush_sg(pages);
33 obj->cache_dirty = false;
36 obj->mm.get_page.sg_pos = pages->sgl;
37 obj->mm.get_page.sg_idx = 0;
39 obj->mm.pages = pages;
41 if (i915_gem_object_is_tiled(obj) &&
42 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
43 GEM_BUG_ON(obj->mm.quirked);
44 __i915_gem_object_pin_pages(obj);
45 obj->mm.quirked = true;
48 GEM_BUG_ON(!sg_page_sizes);
49 obj->mm.page_sizes.phys = sg_page_sizes;
52 * Calculate the supported page-sizes which fit into the given
53 * sg_page_sizes. This will give us the page-sizes which we may be able
54 * to use opportunistically when later inserting into the GTT. For
55 * example if phys=2G, then in theory we should be able to use 1G, 2M,
56 * 64K or 4K pages, although in practice this will depend on a number of
59 obj->mm.page_sizes.sg = 0;
60 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
61 if (obj->mm.page_sizes.phys & ~0u << i)
62 obj->mm.page_sizes.sg |= BIT(i);
64 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
66 if (i915_gem_object_is_shrinkable(obj)) {
67 struct list_head *list;
70 spin_lock_irqsave(&i915->mm.obj_lock, flags);
72 i915->mm.shrink_count++;
73 i915->mm.shrink_memory += obj->base.size;
75 if (obj->mm.madv != I915_MADV_WILLNEED)
76 list = &i915->mm.purge_list;
78 list = &i915->mm.shrink_list;
79 list_add_tail(&obj->mm.link, list);
81 atomic_set(&obj->mm.shrink_pin, 0);
82 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
86 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
88 struct drm_i915_private *i915 = to_i915(obj->base.dev);
91 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
93 "Attempting to obtain a purgeable object\n");
97 err = obj->ops->get_pages(obj);
98 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
103 /* Ensure that the associated pages are gathered from the backing storage
104 * and pinned into our object. i915_gem_object_pin_pages() may be called
105 * multiple times before they are released by a single call to
106 * i915_gem_object_unpin_pages() - once the pages are no longer referenced
107 * either as a result of memory pressure (reaping pages under the shrinker)
108 * or as the object is itself released.
110 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
114 err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES);
118 if (unlikely(!i915_gem_object_has_pages(obj))) {
119 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
121 err = ____i915_gem_object_get_pages(obj);
125 smp_mb__before_atomic();
127 atomic_inc(&obj->mm.pages_pin_count);
130 mutex_unlock(&obj->mm.lock);
134 /* Immediately discard the backing storage */
135 void i915_gem_object_truncate(struct drm_i915_gem_object *obj)
137 drm_gem_free_mmap_offset(&obj->base);
138 if (obj->ops->truncate)
139 obj->ops->truncate(obj);
142 /* Try to discard unwanted pages */
143 void i915_gem_object_writeback(struct drm_i915_gem_object *obj)
145 lockdep_assert_held(&obj->mm.lock);
146 GEM_BUG_ON(i915_gem_object_has_pages(obj));
148 if (obj->ops->writeback)
149 obj->ops->writeback(obj);
152 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
154 struct radix_tree_iter iter;
158 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
159 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
163 static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)
165 if (is_vmalloc_addr(ptr))
170 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
172 struct sg_table *pages;
174 pages = fetch_and_zero(&obj->mm.pages);
175 if (IS_ERR_OR_NULL(pages))
178 if (i915_gem_object_is_volatile(obj))
179 obj->mm.madv = I915_MADV_WILLNEED;
181 i915_gem_object_make_unshrinkable(obj);
183 if (obj->mm.mapping) {
184 unmap_object(obj, page_mask_bits(obj->mm.mapping));
185 obj->mm.mapping = NULL;
188 __i915_gem_object_reset_page_iter(obj);
189 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
191 if (test_and_clear_bit(I915_BO_WAS_BOUND_BIT, &obj->flags)) {
192 struct drm_i915_private *i915 = to_i915(obj->base.dev);
193 intel_wakeref_t wakeref;
195 with_intel_runtime_pm_if_active(&i915->runtime_pm, wakeref)
196 intel_gt_invalidate_tlbs(&i915->gt);
202 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
204 struct sg_table *pages;
207 if (i915_gem_object_has_pinned_pages(obj))
210 /* May be called by shrinker from within get_pages() (on another bo) */
211 mutex_lock(&obj->mm.lock);
212 if (unlikely(atomic_read(&obj->mm.pages_pin_count))) {
217 i915_gem_object_release_mmap_offset(obj);
220 * ->put_pages might need to allocate memory for the bit17 swizzle
221 * array, hence protect them from being reaped by removing them from gtt
224 pages = __i915_gem_object_unset_pages(obj);
227 * XXX Temporary hijinx to avoid updating all backends to handle
228 * NULL pages. In the future, when we have more asynchronous
229 * get_pages backends we should be better able to handle the
230 * cancellation of the async task in a more uniform manner.
232 if (!pages && !i915_gem_object_needs_async_cancel(obj))
233 pages = ERR_PTR(-EINVAL);
236 obj->ops->put_pages(obj, pages);
240 mutex_unlock(&obj->mm.lock);
245 /* The 'mapping' part of i915_gem_object_pin_map() below */
246 static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj,
247 enum i915_map_type type)
249 unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;
250 struct page *stack[32], **pages = stack, *page;
251 struct sgt_iter iter;
258 fallthrough; /* to use PAGE_KERNEL anyway */
261 * On 32b, highmem using a finite set of indirect PTE (i.e.
262 * vmap) to provide virtual mappings of the high pages.
263 * As these are finite, map_new_virtual() must wait for some
264 * other kmap() to finish when it runs out. If we map a large
265 * number of objects, there is no method for it to tell us
266 * to release the mappings, and we deadlock.
268 * However, if we make an explicit vmap of the page, that
269 * uses a larger vmalloc arena, and also has the ability
270 * to tell us to release unwanted mappings. Most importantly,
271 * it will fail and propagate an error instead of waiting
274 * So if the page is beyond the 32b boundary, make an explicit
277 if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl)))
278 return page_address(sg_page(obj->mm.pages->sgl));
279 pgprot = PAGE_KERNEL;
282 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
286 if (n_pages > ARRAY_SIZE(stack)) {
287 /* Too big for stack -- allocate temporary array instead */
288 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
294 for_each_sgt_page(page, iter, obj->mm.pages)
296 vaddr = vmap(pages, n_pages, 0, pgprot);
302 static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj,
303 enum i915_map_type type)
305 resource_size_t iomap = obj->mm.region->iomap.base -
306 obj->mm.region->region.start;
307 unsigned long n_pfn = obj->base.size >> PAGE_SHIFT;
308 unsigned long stack[32], *pfns = stack, i;
309 struct sgt_iter iter;
313 if (type != I915_MAP_WC)
316 if (n_pfn > ARRAY_SIZE(stack)) {
317 /* Too big for stack -- allocate temporary array instead */
318 pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL);
324 for_each_sgt_daddr(addr, iter, obj->mm.pages)
325 pfns[i++] = (iomap + addr) >> PAGE_SHIFT;
326 vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO));
332 /* get, pin, and map the pages of the object into kernel space */
333 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
334 enum i915_map_type type)
336 enum i915_map_type has_type;
342 flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | I915_GEM_OBJECT_HAS_IOMEM;
343 if (!i915_gem_object_type_has(obj, flags))
344 return ERR_PTR(-ENXIO);
346 err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES);
350 pinned = !(type & I915_MAP_OVERRIDE);
351 type &= ~I915_MAP_OVERRIDE;
353 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
354 if (unlikely(!i915_gem_object_has_pages(obj))) {
355 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
357 err = ____i915_gem_object_get_pages(obj);
361 smp_mb__before_atomic();
363 atomic_inc(&obj->mm.pages_pin_count);
366 GEM_BUG_ON(!i915_gem_object_has_pages(obj));
368 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
369 if (ptr && has_type != type) {
375 unmap_object(obj, ptr);
377 ptr = obj->mm.mapping = NULL;
381 if (GEM_WARN_ON(type == I915_MAP_WC &&
382 !static_cpu_has(X86_FEATURE_PAT)))
384 else if (i915_gem_object_has_struct_page(obj))
385 ptr = i915_gem_object_map_page(obj, type);
387 ptr = i915_gem_object_map_pfn(obj, type);
393 obj->mm.mapping = page_pack_bits(ptr, type);
397 mutex_unlock(&obj->mm.lock);
401 atomic_dec(&obj->mm.pages_pin_count);
407 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
408 unsigned long offset,
411 enum i915_map_type has_type;
414 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
415 GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),
416 offset, size, obj->base.size));
418 wmb(); /* let all previous writes be visible to coherent partners */
419 obj->mm.dirty = true;
421 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)
424 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
425 if (has_type == I915_MAP_WC)
428 drm_clflush_virt_range(ptr + offset, size);
429 if (size == obj->base.size) {
430 obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
431 obj->cache_dirty = false;
435 void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)
437 GEM_BUG_ON(!obj->mm.mapping);
440 * We allow removing the mapping from underneath pinned pages!
442 * Furthermore, since this is an unsafe operation reserved only
443 * for construction time manipulation, we ignore locking prudence.
445 unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping)));
447 i915_gem_object_unpin_map(obj);
451 i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
453 unsigned int *offset)
455 struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
456 struct scatterlist *sg;
457 unsigned int idx, count;
460 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
461 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
463 /* As we iterate forward through the sg, we record each entry in a
464 * radixtree for quick repeated (backwards) lookups. If we have seen
465 * this index previously, we will have an entry for it.
467 * Initial lookup is O(N), but this is amortized to O(1) for
468 * sequential page access (where each new request is consecutive
469 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
470 * i.e. O(1) with a large constant!
472 if (n < READ_ONCE(iter->sg_idx))
475 mutex_lock(&iter->lock);
477 /* We prefer to reuse the last sg so that repeated lookup of this
478 * (or the subsequent) sg are fast - comparing against the last
479 * sg is faster than going through the radixtree.
484 count = __sg_page_count(sg);
486 while (idx + count <= n) {
491 /* If we cannot allocate and insert this entry, or the
492 * individual pages from this range, cancel updating the
493 * sg_idx so that on this lookup we are forced to linearly
494 * scan onwards, but on future lookups we will try the
495 * insertion again (in which case we need to be careful of
496 * the error return reporting that we have already inserted
499 ret = radix_tree_insert(&iter->radix, idx, sg);
500 if (ret && ret != -EEXIST)
503 entry = xa_mk_value(idx);
504 for (i = 1; i < count; i++) {
505 ret = radix_tree_insert(&iter->radix, idx + i, entry);
506 if (ret && ret != -EEXIST)
511 sg = ____sg_next(sg);
512 count = __sg_page_count(sg);
519 mutex_unlock(&iter->lock);
521 if (unlikely(n < idx)) /* insertion completed by another thread */
524 /* In case we failed to insert the entry into the radixtree, we need
525 * to look beyond the current sg.
527 while (idx + count <= n) {
529 sg = ____sg_next(sg);
530 count = __sg_page_count(sg);
539 sg = radix_tree_lookup(&iter->radix, n);
542 /* If this index is in the middle of multi-page sg entry,
543 * the radix tree will contain a value entry that points
544 * to the start of that range. We will return the pointer to
545 * the base page and the offset of this page within the
549 if (unlikely(xa_is_value(sg))) {
550 unsigned long base = xa_to_value(sg);
552 sg = radix_tree_lookup(&iter->radix, base);
564 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
566 struct scatterlist *sg;
569 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
571 sg = i915_gem_object_get_sg(obj, n, &offset);
572 return nth_page(sg_page(sg), offset);
575 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
577 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
582 page = i915_gem_object_get_page(obj, n);
584 set_page_dirty(page);
590 i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
594 struct scatterlist *sg;
597 sg = i915_gem_object_get_sg(obj, n, &offset);
600 *len = sg_dma_len(sg) - (offset << PAGE_SHIFT);
602 return sg_dma_address(sg) + (offset << PAGE_SHIFT);
606 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
609 return i915_gem_object_get_dma_address_len(obj, n, NULL);