2 * SPDX-License-Identifier: MIT
4 * Copyright © 2016-2018 Intel Corporation
9 #include "i915_active.h"
10 #include "i915_syncmap.h"
12 #include "intel_ring.h"
13 #include "intel_timeline.h"
15 #define ptr_set_bit(ptr, bit) ((typeof(ptr))((unsigned long)(ptr) | BIT(bit)))
16 #define ptr_test_bit(ptr, bit) ((unsigned long)(ptr) & BIT(bit))
18 #define CACHELINE_BITS 6
19 #define CACHELINE_FREE CACHELINE_BITS
21 struct intel_timeline_hwsp {
23 struct intel_gt_timelines *gt_timelines;
24 struct list_head free_link;
29 static struct i915_vma *__hwsp_alloc(struct intel_gt *gt)
31 struct drm_i915_private *i915 = gt->i915;
32 struct drm_i915_gem_object *obj;
35 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
39 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
41 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
43 i915_gem_object_put(obj);
48 static struct i915_vma *
49 hwsp_alloc(struct intel_timeline *timeline, unsigned int *cacheline)
51 struct intel_gt_timelines *gt = &timeline->gt->timelines;
52 struct intel_timeline_hwsp *hwsp;
54 BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
56 spin_lock_irq(>->hwsp_lock);
58 /* hwsp_free_list only contains HWSP that have available cachelines */
59 hwsp = list_first_entry_or_null(>->hwsp_free_list,
60 typeof(*hwsp), free_link);
64 spin_unlock_irq(>->hwsp_lock);
66 hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
68 return ERR_PTR(-ENOMEM);
70 vma = __hwsp_alloc(timeline->gt);
76 GT_TRACE(timeline->gt, "new HWSP allocated\n");
79 hwsp->gt = timeline->gt;
81 hwsp->free_bitmap = ~0ull;
82 hwsp->gt_timelines = gt;
84 spin_lock_irq(>->hwsp_lock);
85 list_add(&hwsp->free_link, >->hwsp_free_list);
88 GEM_BUG_ON(!hwsp->free_bitmap);
89 *cacheline = __ffs64(hwsp->free_bitmap);
90 hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
91 if (!hwsp->free_bitmap)
92 list_del(&hwsp->free_link);
94 spin_unlock_irq(>->hwsp_lock);
96 GEM_BUG_ON(hwsp->vma->private != hwsp);
100 static void __idle_hwsp_free(struct intel_timeline_hwsp *hwsp, int cacheline)
102 struct intel_gt_timelines *gt = hwsp->gt_timelines;
105 spin_lock_irqsave(>->hwsp_lock, flags);
107 /* As a cacheline becomes available, publish the HWSP on the freelist */
108 if (!hwsp->free_bitmap)
109 list_add_tail(&hwsp->free_link, >->hwsp_free_list);
111 GEM_BUG_ON(cacheline >= BITS_PER_TYPE(hwsp->free_bitmap));
112 hwsp->free_bitmap |= BIT_ULL(cacheline);
114 /* And if no one is left using it, give the page back to the system */
115 if (hwsp->free_bitmap == ~0ull) {
116 i915_vma_put(hwsp->vma);
117 list_del(&hwsp->free_link);
121 spin_unlock_irqrestore(>->hwsp_lock, flags);
124 static void __rcu_cacheline_free(struct rcu_head *rcu)
126 struct intel_timeline_cacheline *cl =
127 container_of(rcu, typeof(*cl), rcu);
129 /* Must wait until after all *rq->hwsp are complete before removing */
130 i915_gem_object_unpin_map(cl->hwsp->vma->obj);
131 __idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS));
133 i915_active_fini(&cl->active);
137 static void __idle_cacheline_free(struct intel_timeline_cacheline *cl)
139 GEM_BUG_ON(!i915_active_is_idle(&cl->active));
140 call_rcu(&cl->rcu, __rcu_cacheline_free);
144 static void __cacheline_retire(struct i915_active *active)
146 struct intel_timeline_cacheline *cl =
147 container_of(active, typeof(*cl), active);
149 i915_vma_unpin(cl->hwsp->vma);
150 if (ptr_test_bit(cl->vaddr, CACHELINE_FREE))
151 __idle_cacheline_free(cl);
154 static int __cacheline_active(struct i915_active *active)
156 struct intel_timeline_cacheline *cl =
157 container_of(active, typeof(*cl), active);
159 __i915_vma_pin(cl->hwsp->vma);
163 static struct intel_timeline_cacheline *
164 cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)
166 struct intel_timeline_cacheline *cl;
169 GEM_BUG_ON(cacheline >= BIT(CACHELINE_BITS));
171 cl = kmalloc(sizeof(*cl), GFP_KERNEL);
173 return ERR_PTR(-ENOMEM);
175 vaddr = i915_gem_object_pin_map(hwsp->vma->obj, I915_MAP_WB);
178 return ERR_CAST(vaddr);
182 cl->vaddr = page_pack_bits(vaddr, cacheline);
184 i915_active_init(&cl->active, __cacheline_active, __cacheline_retire);
189 static void cacheline_acquire(struct intel_timeline_cacheline *cl,
195 cl->ggtt_offset = ggtt_offset;
196 i915_active_acquire(&cl->active);
199 static void cacheline_release(struct intel_timeline_cacheline *cl)
202 i915_active_release(&cl->active);
205 static void cacheline_free(struct intel_timeline_cacheline *cl)
207 if (!i915_active_acquire_if_busy(&cl->active)) {
208 __idle_cacheline_free(cl);
212 GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE));
213 cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE);
215 i915_active_release(&cl->active);
218 static int intel_timeline_init(struct intel_timeline *timeline,
220 struct i915_vma *hwsp,
225 kref_init(&timeline->kref);
226 atomic_set(&timeline->pin_count, 0);
230 timeline->has_initial_breadcrumb = !hwsp;
231 timeline->hwsp_cacheline = NULL;
234 struct intel_timeline_cacheline *cl;
235 unsigned int cacheline;
237 hwsp = hwsp_alloc(timeline, &cacheline);
239 return PTR_ERR(hwsp);
241 cl = cacheline_alloc(hwsp->private, cacheline);
243 __idle_hwsp_free(hwsp->private, cacheline);
247 timeline->hwsp_cacheline = cl;
248 timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
250 vaddr = page_mask_bits(cl->vaddr);
252 timeline->hwsp_offset = offset;
253 vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
255 return PTR_ERR(vaddr);
258 timeline->hwsp_seqno =
259 memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);
261 timeline->hwsp_ggtt = i915_vma_get(hwsp);
262 GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);
264 timeline->fence_context = dma_fence_context_alloc(1);
266 mutex_init(&timeline->mutex);
268 INIT_ACTIVE_FENCE(&timeline->last_request);
269 INIT_LIST_HEAD(&timeline->requests);
271 i915_syncmap_init(&timeline->sync);
276 void intel_gt_init_timelines(struct intel_gt *gt)
278 struct intel_gt_timelines *timelines = >->timelines;
280 spin_lock_init(&timelines->lock);
281 INIT_LIST_HEAD(&timelines->active_list);
283 spin_lock_init(&timelines->hwsp_lock);
284 INIT_LIST_HEAD(&timelines->hwsp_free_list);
287 static void intel_timeline_fini(struct intel_timeline *timeline)
289 GEM_BUG_ON(atomic_read(&timeline->pin_count));
290 GEM_BUG_ON(!list_empty(&timeline->requests));
291 GEM_BUG_ON(timeline->retire);
293 if (timeline->hwsp_cacheline)
294 cacheline_free(timeline->hwsp_cacheline);
296 i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
298 i915_vma_put(timeline->hwsp_ggtt);
301 * A small race exists between intel_gt_retire_requests_timeout and
302 * intel_timeline_exit which could result in the syncmap not getting
303 * free'd. Rather than work to hard to seal this race, simply cleanup
304 * the syncmap on fini.
306 i915_syncmap_free(&timeline->sync);
309 struct intel_timeline *
310 __intel_timeline_create(struct intel_gt *gt,
311 struct i915_vma *global_hwsp,
314 struct intel_timeline *timeline;
317 timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
319 return ERR_PTR(-ENOMEM);
321 err = intel_timeline_init(timeline, gt, global_hwsp, offset);
330 void __intel_timeline_pin(struct intel_timeline *tl)
332 GEM_BUG_ON(!atomic_read(&tl->pin_count));
333 atomic_inc(&tl->pin_count);
336 int intel_timeline_pin(struct intel_timeline *tl, struct i915_gem_ww_ctx *ww)
340 if (atomic_add_unless(&tl->pin_count, 1, 0))
343 err = i915_ggtt_pin(tl->hwsp_ggtt, ww, 0, PIN_HIGH);
348 i915_ggtt_offset(tl->hwsp_ggtt) +
349 offset_in_page(tl->hwsp_offset);
350 GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
351 tl->fence_context, tl->hwsp_offset);
353 cacheline_acquire(tl->hwsp_cacheline, tl->hwsp_offset);
354 if (atomic_fetch_inc(&tl->pin_count)) {
355 cacheline_release(tl->hwsp_cacheline);
356 __i915_vma_unpin(tl->hwsp_ggtt);
362 void intel_timeline_reset_seqno(const struct intel_timeline *tl)
364 /* Must be pinned to be writable, and no requests in flight. */
365 GEM_BUG_ON(!atomic_read(&tl->pin_count));
366 WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);
369 void intel_timeline_enter(struct intel_timeline *tl)
371 struct intel_gt_timelines *timelines = &tl->gt->timelines;
374 * Pretend we are serialised by the timeline->mutex.
376 * While generally true, there are a few exceptions to the rule
377 * for the engine->kernel_context being used to manage power
378 * transitions. As the engine_park may be called from under any
379 * timeline, it uses the power mutex as a global serialisation
380 * lock to prevent any other request entering its timeline.
382 * The rule is generally tl->mutex, otherwise engine->wakeref.mutex.
384 * However, intel_gt_retire_request() does not know which engine
385 * it is retiring along and so cannot partake in the engine-pm
386 * barrier, and there we use the tl->active_count as a means to
387 * pin the timeline in the active_list while the locks are dropped.
388 * Ergo, as that is outside of the engine-pm barrier, we need to
389 * use atomic to manipulate tl->active_count.
391 lockdep_assert_held(&tl->mutex);
393 if (atomic_add_unless(&tl->active_count, 1, 0))
396 spin_lock(&timelines->lock);
397 if (!atomic_fetch_inc(&tl->active_count)) {
399 * The HWSP is volatile, and may have been lost while inactive,
400 * e.g. across suspend/resume. Be paranoid, and ensure that
401 * the HWSP value matches our seqno so we don't proclaim
402 * the next request as already complete.
404 intel_timeline_reset_seqno(tl);
405 list_add_tail(&tl->link, &timelines->active_list);
407 spin_unlock(&timelines->lock);
410 void intel_timeline_exit(struct intel_timeline *tl)
412 struct intel_gt_timelines *timelines = &tl->gt->timelines;
414 /* See intel_timeline_enter() */
415 lockdep_assert_held(&tl->mutex);
417 GEM_BUG_ON(!atomic_read(&tl->active_count));
418 if (atomic_add_unless(&tl->active_count, -1, 1))
421 spin_lock(&timelines->lock);
422 if (atomic_dec_and_test(&tl->active_count))
424 spin_unlock(&timelines->lock);
427 * Since this timeline is idle, all bariers upon which we were waiting
428 * must also be complete and so we can discard the last used barriers
429 * without loss of information.
431 i915_syncmap_free(&tl->sync);
434 static u32 timeline_advance(struct intel_timeline *tl)
436 GEM_BUG_ON(!atomic_read(&tl->pin_count));
437 GEM_BUG_ON(tl->seqno & tl->has_initial_breadcrumb);
439 return tl->seqno += 1 + tl->has_initial_breadcrumb;
442 static void timeline_rollback(struct intel_timeline *tl)
444 tl->seqno -= 1 + tl->has_initial_breadcrumb;
448 __intel_timeline_get_seqno(struct intel_timeline *tl,
449 struct i915_request *rq,
452 struct intel_timeline_cacheline *cl;
453 unsigned int cacheline;
454 struct i915_vma *vma;
458 might_lock(&tl->gt->ggtt->vm.mutex);
459 GT_TRACE(tl->gt, "timeline:%llx wrapped\n", tl->fence_context);
462 * If there is an outstanding GPU reference to this cacheline,
463 * such as it being sampled by a HW semaphore on another timeline,
464 * we cannot wraparound our seqno value (the HW semaphore does
465 * a strict greater-than-or-equals compare, not i915_seqno_passed).
466 * So if the cacheline is still busy, we must detach ourselves
467 * from it and leave it inflight alongside its users.
469 * However, if nobody is watching and we can guarantee that nobody
470 * will, we could simply reuse the same cacheline.
472 * if (i915_active_request_is_signaled(&tl->last_request) &&
473 * i915_active_is_signaled(&tl->hwsp_cacheline->active))
476 * That seems unlikely for a busy timeline that needed to wrap in
477 * the first place, so just replace the cacheline.
480 vma = hwsp_alloc(tl, &cacheline);
486 err = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
488 __idle_hwsp_free(vma->private, cacheline);
492 cl = cacheline_alloc(vma->private, cacheline);
495 __idle_hwsp_free(vma->private, cacheline);
498 GEM_BUG_ON(cl->hwsp->vma != vma);
501 * Attach the old cacheline to the current request, so that we only
502 * free it after the current request is retired, which ensures that
503 * all writes into the cacheline from previous requests are complete.
505 err = i915_active_ref(&tl->hwsp_cacheline->active,
511 cacheline_release(tl->hwsp_cacheline); /* ownership now xfered to rq */
512 cacheline_free(tl->hwsp_cacheline);
514 i915_vma_unpin(tl->hwsp_ggtt); /* binding kept alive by old cacheline */
515 i915_vma_put(tl->hwsp_ggtt);
517 tl->hwsp_ggtt = i915_vma_get(vma);
519 vaddr = page_mask_bits(cl->vaddr);
520 tl->hwsp_offset = cacheline * CACHELINE_BYTES;
522 memset(vaddr + tl->hwsp_offset, 0, CACHELINE_BYTES);
524 tl->hwsp_offset += i915_ggtt_offset(vma);
525 GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
526 tl->fence_context, tl->hwsp_offset);
528 cacheline_acquire(cl, tl->hwsp_offset);
529 tl->hwsp_cacheline = cl;
531 *seqno = timeline_advance(tl);
532 GEM_BUG_ON(i915_seqno_passed(*tl->hwsp_seqno, *seqno));
540 timeline_rollback(tl);
544 int intel_timeline_get_seqno(struct intel_timeline *tl,
545 struct i915_request *rq,
548 *seqno = timeline_advance(tl);
550 /* Replace the HWSP on wraparound for HW semaphores */
551 if (unlikely(!*seqno && tl->hwsp_cacheline))
552 return __intel_timeline_get_seqno(tl, rq, seqno);
557 static int cacheline_ref(struct intel_timeline_cacheline *cl,
558 struct i915_request *rq)
560 return i915_active_add_request(&cl->active, rq);
563 int intel_timeline_read_hwsp(struct i915_request *from,
564 struct i915_request *to,
567 struct intel_timeline_cacheline *cl;
570 GEM_BUG_ON(!rcu_access_pointer(from->hwsp_cacheline));
573 cl = rcu_dereference(from->hwsp_cacheline);
574 if (i915_request_completed(from)) /* confirm cacheline is valid */
576 if (unlikely(!i915_active_acquire_if_busy(&cl->active)))
577 goto unlock; /* seqno wrapped and completed! */
578 if (unlikely(i915_request_completed(from)))
582 err = cacheline_ref(cl, to);
586 *hwsp = cl->ggtt_offset;
588 i915_active_release(&cl->active);
592 i915_active_release(&cl->active);
598 void intel_timeline_unpin(struct intel_timeline *tl)
600 GEM_BUG_ON(!atomic_read(&tl->pin_count));
601 if (!atomic_dec_and_test(&tl->pin_count))
604 cacheline_release(tl->hwsp_cacheline);
606 __i915_vma_unpin(tl->hwsp_ggtt);
609 void __intel_timeline_free(struct kref *kref)
611 struct intel_timeline *timeline =
612 container_of(kref, typeof(*timeline), kref);
614 intel_timeline_fini(timeline);
615 kfree_rcu(timeline, rcu);
618 void intel_gt_fini_timelines(struct intel_gt *gt)
620 struct intel_gt_timelines *timelines = >->timelines;
622 GEM_BUG_ON(!list_empty(&timelines->active_list));
623 GEM_BUG_ON(!list_empty(&timelines->hwsp_free_list));
626 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
627 #include "gt/selftests/mock_timeline.c"
628 #include "gt/selftest_timeline.c"