2 * SPDX-License-Identifier: MIT
4 * Copyright © 2014-2016 Intel Corporation
7 #include "display/intel_frontbuffer.h"
10 #include "i915_gem_clflush.h"
11 #include "i915_gem_gtt.h"
12 #include "i915_gem_ioctls.h"
13 #include "i915_gem_object.h"
15 #include "i915_gem_lmem.h"
16 #include "i915_gem_mman.h"
18 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
21 * We manually flush the CPU domain so that we can override and
22 * force the flush for the display, and perform it asyncrhonously.
24 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
26 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
27 obj->write_domain = 0;
30 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
32 if (!i915_gem_object_is_framebuffer(obj))
35 i915_gem_object_lock(obj, NULL);
36 __i915_gem_object_flush_for_display(obj);
37 i915_gem_object_unlock(obj);
40 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj)
42 if (i915_gem_object_is_framebuffer(obj))
43 __i915_gem_object_flush_for_display(obj);
47 * Moves a single object to the WC read, and possibly write domain.
48 * @obj: object to act on
49 * @write: ask for write access or read only
51 * This function returns when the move is complete, including waiting on
55 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
59 assert_object_held(obj);
61 ret = i915_gem_object_wait(obj,
62 I915_WAIT_INTERRUPTIBLE |
63 (write ? I915_WAIT_ALL : 0),
64 MAX_SCHEDULE_TIMEOUT);
68 if (obj->write_domain == I915_GEM_DOMAIN_WC)
71 /* Flush and acquire obj->pages so that we are coherent through
72 * direct access in memory with previous cached writes through
73 * shmemfs and that our cache domain tracking remains valid.
74 * For example, if the obj->filp was moved to swap without us
75 * being notified and releasing the pages, we would mistakenly
76 * continue to assume that the obj remained out of the CPU cached
79 ret = i915_gem_object_pin_pages(obj);
83 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
85 /* Serialise direct access to this object with the barriers for
86 * coherent writes from the GPU, by effectively invalidating the
87 * WC domain upon first access.
89 if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
92 /* It should now be out of any other write domains, and we can update
93 * the domain values for our changes.
95 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
96 obj->read_domains |= I915_GEM_DOMAIN_WC;
98 obj->read_domains = I915_GEM_DOMAIN_WC;
99 obj->write_domain = I915_GEM_DOMAIN_WC;
100 obj->mm.dirty = true;
103 i915_gem_object_unpin_pages(obj);
108 * Moves a single object to the GTT read, and possibly write domain.
109 * @obj: object to act on
110 * @write: ask for write access or read only
112 * This function returns when the move is complete, including waiting on
116 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
120 assert_object_held(obj);
122 ret = i915_gem_object_wait(obj,
123 I915_WAIT_INTERRUPTIBLE |
124 (write ? I915_WAIT_ALL : 0),
125 MAX_SCHEDULE_TIMEOUT);
129 if (obj->write_domain == I915_GEM_DOMAIN_GTT)
132 /* Flush and acquire obj->pages so that we are coherent through
133 * direct access in memory with previous cached writes through
134 * shmemfs and that our cache domain tracking remains valid.
135 * For example, if the obj->filp was moved to swap without us
136 * being notified and releasing the pages, we would mistakenly
137 * continue to assume that the obj remained out of the CPU cached
140 ret = i915_gem_object_pin_pages(obj);
144 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
146 /* Serialise direct access to this object with the barriers for
147 * coherent writes from the GPU, by effectively invalidating the
148 * GTT domain upon first access.
150 if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
153 /* It should now be out of any other write domains, and we can update
154 * the domain values for our changes.
156 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
157 obj->read_domains |= I915_GEM_DOMAIN_GTT;
159 struct i915_vma *vma;
161 obj->read_domains = I915_GEM_DOMAIN_GTT;
162 obj->write_domain = I915_GEM_DOMAIN_GTT;
163 obj->mm.dirty = true;
165 spin_lock(&obj->vma.lock);
166 for_each_ggtt_vma(vma, obj)
167 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
168 i915_vma_set_ggtt_write(vma);
169 spin_unlock(&obj->vma.lock);
172 i915_gem_object_unpin_pages(obj);
177 * Changes the cache-level of an object across all VMA.
178 * @obj: object to act on
179 * @cache_level: new cache level to set for the object
181 * After this function returns, the object will be in the new cache-level
182 * across all GTT and the contents of the backing storage will be coherent,
183 * with respect to the new cache-level. In order to keep the backing storage
184 * coherent for all users, we only allow a single cache level to be set
185 * globally on the object and prevent it from being changed whilst the
186 * hardware is reading from the object. That is if the object is currently
187 * on the scanout it will be set to uncached (or equivalent display
188 * cache coherency) and all non-MOCS GPU access will also be uncached so
189 * that all direct access to the scanout remains coherent.
191 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
192 enum i915_cache_level cache_level)
196 if (obj->cache_level == cache_level)
199 ret = i915_gem_object_wait(obj,
200 I915_WAIT_INTERRUPTIBLE |
202 MAX_SCHEDULE_TIMEOUT);
206 /* Always invalidate stale cachelines */
207 if (obj->cache_level != cache_level) {
208 i915_gem_object_set_cache_coherency(obj, cache_level);
209 obj->cache_dirty = true;
212 /* The cache-level will be applied when each vma is rebound. */
213 return i915_gem_object_unbind(obj,
214 I915_GEM_OBJECT_UNBIND_ACTIVE |
215 I915_GEM_OBJECT_UNBIND_BARRIER);
218 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
219 struct drm_file *file)
221 struct drm_i915_gem_caching *args = data;
222 struct drm_i915_gem_object *obj;
226 obj = i915_gem_object_lookup_rcu(file, args->handle);
232 switch (obj->cache_level) {
234 case I915_CACHE_L3_LLC:
235 args->caching = I915_CACHING_CACHED;
239 args->caching = I915_CACHING_DISPLAY;
243 args->caching = I915_CACHING_NONE;
251 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
252 struct drm_file *file)
254 struct drm_i915_private *i915 = to_i915(dev);
255 struct drm_i915_gem_caching *args = data;
256 struct drm_i915_gem_object *obj;
257 enum i915_cache_level level;
260 switch (args->caching) {
261 case I915_CACHING_NONE:
262 level = I915_CACHE_NONE;
264 case I915_CACHING_CACHED:
266 * Due to a HW issue on BXT A stepping, GPU stores via a
267 * snooped mapping may leave stale data in a corresponding CPU
268 * cacheline, whereas normally such cachelines would get
271 if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
274 level = I915_CACHE_LLC;
276 case I915_CACHING_DISPLAY:
277 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
283 obj = i915_gem_object_lookup(file, args->handle);
288 * The caching mode of proxy object is handled by its generator, and
289 * not allowed to be changed by userspace.
291 if (i915_gem_object_is_proxy(obj)) {
296 ret = i915_gem_object_lock_interruptible(obj, NULL);
300 ret = i915_gem_object_set_cache_level(obj, level);
301 i915_gem_object_unlock(obj);
304 i915_gem_object_put(obj);
309 * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
310 * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
311 * (for pageflips). We only flush the caches while preparing the buffer for
312 * display, the callers are responsible for frontbuffer flush.
315 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
317 const struct i915_ggtt_view *view,
320 struct drm_i915_private *i915 = to_i915(obj->base.dev);
321 struct i915_gem_ww_ctx ww;
322 struct i915_vma *vma;
325 /* Frame buffer must be in LMEM (no migration yet) */
326 if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))
327 return ERR_PTR(-EINVAL);
329 i915_gem_ww_ctx_init(&ww, true);
331 ret = i915_gem_object_lock(obj, &ww);
335 * The display engine is not coherent with the LLC cache on gen6. As
336 * a result, we make sure that the pinning that is about to occur is
337 * done with uncached PTEs. This is lowest common denominator for all
340 * However for gen6+, we could do better by using the GFDT bit instead
341 * of uncaching, which would allow us to flush all the LLC-cached data
342 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
344 ret = i915_gem_object_set_cache_level(obj,
346 I915_CACHE_WT : I915_CACHE_NONE);
351 * As the user may map the buffer once pinned in the display plane
352 * (e.g. libkms for the bootup splash), we have to ensure that we
353 * always use map_and_fenceable for all scanout buffers. However,
354 * it may simply be too big to fit into mappable, in which case
355 * put it anyway and hope that userspace can cope (but always first
356 * try to preserve the existing ABI).
358 vma = ERR_PTR(-ENOSPC);
359 if ((flags & PIN_MAPPABLE) == 0 &&
360 (!view || view->type == I915_GGTT_VIEW_NORMAL))
361 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0, alignment,
362 flags | PIN_MAPPABLE |
364 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK))
365 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0,
372 vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
374 i915_gem_object_flush_if_display_locked(obj);
377 if (ret == -EDEADLK) {
378 ret = i915_gem_ww_ctx_backoff(&ww);
382 i915_gem_ww_ctx_fini(&ww);
391 * Moves a single object to the CPU read, and possibly write domain.
392 * @obj: object to act on
393 * @write: requesting write or read-only access
395 * This function returns when the move is complete, including waiting on
399 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
403 assert_object_held(obj);
405 ret = i915_gem_object_wait(obj,
406 I915_WAIT_INTERRUPTIBLE |
407 (write ? I915_WAIT_ALL : 0),
408 MAX_SCHEDULE_TIMEOUT);
412 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
414 /* Flush the CPU cache if it's still invalid. */
415 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
416 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
417 obj->read_domains |= I915_GEM_DOMAIN_CPU;
420 /* It should now be out of any other write domains, and we can update
421 * the domain values for our changes.
423 GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
425 /* If we're writing through the CPU, then the GPU read domains will
426 * need to be invalidated at next use.
429 __start_cpu_write(obj);
435 * Called when user space prepares to use an object with the CPU, either
436 * through the mmap ioctl's mapping or a GTT mapping.
438 * @data: ioctl data blob
442 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
443 struct drm_file *file)
445 struct drm_i915_gem_set_domain *args = data;
446 struct drm_i915_gem_object *obj;
447 u32 read_domains = args->read_domains;
448 u32 write_domain = args->write_domain;
451 /* Only handle setting domains to types used by the CPU. */
452 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
456 * Having something in the write domain implies it's in the read
457 * domain, and only that read domain. Enforce that in the request.
459 if (write_domain && read_domains != write_domain)
465 obj = i915_gem_object_lookup(file, args->handle);
470 * Try to flush the object off the GPU without holding the lock.
471 * We will repeat the flush holding the lock in the normal manner
472 * to catch cases where we are gazumped.
474 err = i915_gem_object_wait(obj,
475 I915_WAIT_INTERRUPTIBLE |
477 (write_domain ? I915_WAIT_ALL : 0),
478 MAX_SCHEDULE_TIMEOUT);
483 * Proxy objects do not control access to the backing storage, ergo
484 * they cannot be used as a means to manipulate the cache domain
485 * tracking for that backing storage. The proxy object is always
486 * considered to be outside of any cache domain.
488 if (i915_gem_object_is_proxy(obj)) {
494 * Flush and acquire obj->pages so that we are coherent through
495 * direct access in memory with previous cached writes through
496 * shmemfs and that our cache domain tracking remains valid.
497 * For example, if the obj->filp was moved to swap without us
498 * being notified and releasing the pages, we would mistakenly
499 * continue to assume that the obj remained out of the CPU cached
502 err = i915_gem_object_pin_pages(obj);
507 * Already in the desired write domain? Nothing for us to do!
509 * We apply a little bit of cunning here to catch a broader set of
510 * no-ops. If obj->write_domain is set, we must be in the same
511 * obj->read_domains, and only that domain. Therefore, if that
512 * obj->write_domain matches the request read_domains, we are
513 * already in the same read/write domain and can skip the operation,
514 * without having to further check the requested write_domain.
516 if (READ_ONCE(obj->write_domain) == read_domains)
519 err = i915_gem_object_lock_interruptible(obj, NULL);
523 if (read_domains & I915_GEM_DOMAIN_WC)
524 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
525 else if (read_domains & I915_GEM_DOMAIN_GTT)
526 err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
528 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
530 i915_gem_object_unlock(obj);
533 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
536 i915_gem_object_unpin_pages(obj);
538 i915_gem_object_put(obj);
543 * Pins the specified object's pages and synchronizes the object with
544 * GPU accesses. Sets needs_clflush to non-zero if the caller should
545 * flush the object from the CPU cache.
547 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
548 unsigned int *needs_clflush)
553 if (!i915_gem_object_has_struct_page(obj))
556 assert_object_held(obj);
558 ret = i915_gem_object_wait(obj,
559 I915_WAIT_INTERRUPTIBLE,
560 MAX_SCHEDULE_TIMEOUT);
564 ret = i915_gem_object_pin_pages(obj);
568 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
569 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
570 ret = i915_gem_object_set_to_cpu_domain(obj, false);
577 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
579 /* If we're not in the cpu read domain, set ourself into the gtt
580 * read domain and manually flush cachelines (if required). This
581 * optimizes for the case when the gpu will dirty the data
582 * anyway again before the next pread happens.
584 if (!obj->cache_dirty &&
585 !(obj->read_domains & I915_GEM_DOMAIN_CPU))
586 *needs_clflush = CLFLUSH_BEFORE;
589 /* return with the pages pinned */
593 i915_gem_object_unpin_pages(obj);
597 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
598 unsigned int *needs_clflush)
603 if (!i915_gem_object_has_struct_page(obj))
606 assert_object_held(obj);
608 ret = i915_gem_object_wait(obj,
609 I915_WAIT_INTERRUPTIBLE |
611 MAX_SCHEDULE_TIMEOUT);
615 ret = i915_gem_object_pin_pages(obj);
619 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
620 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
621 ret = i915_gem_object_set_to_cpu_domain(obj, true);
628 i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
630 /* If we're not in the cpu write domain, set ourself into the
631 * gtt write domain and manually flush cachelines (as required).
632 * This optimizes for the case when the gpu will use the data
633 * right away and we therefore have to clflush anyway.
635 if (!obj->cache_dirty) {
636 *needs_clflush |= CLFLUSH_AFTER;
639 * Same trick applies to invalidate partially written
640 * cachelines read before writing.
642 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
643 *needs_clflush |= CLFLUSH_BEFORE;
647 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
648 obj->mm.dirty = true;
649 /* return with the pages pinned */
653 i915_gem_object_unpin_pages(obj);