drivers/gpu/drm/i915/gt/intel_timeline.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2016-2018 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8
   9 #include "i915_active.h"
  10 #include "i915_syncmap.h"
  11 #include "intel_gt.h"
  12 #include "intel_ring.h"
  13 #include "intel_timeline.h"
  14
  15 #define ptr_set_bit(ptr, bit) ((typeof(ptr))((unsigned long)(ptr) | BIT(bit)))
  16 #define ptr_test_bit(ptr, bit) ((unsigned long)(ptr) & BIT(bit))
  17
  18 #define CACHELINE_BITS 6
  19 #define CACHELINE_FREE CACHELINE_BITS
  20
  21 struct intel_timeline_hwsp {
  22         struct intel_gt *gt;
  23         struct intel_gt_timelines *gt_timelines;
  24         struct list_head free_link;
  25         struct i915_vma *vma;
  26         u64 free_bitmap;
  27 };
  28
  29 static struct i915_vma *__hwsp_alloc(struct intel_gt *gt)
  30 {
  31         struct drm_i915_private *i915 = gt->i915;
  32         struct drm_i915_gem_object *obj;
  33         struct i915_vma *vma;
  34
  35         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
  36         if (IS_ERR(obj))
  37                 return ERR_CAST(obj);
  38
  39         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
  40
  41         vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
  42         if (IS_ERR(vma))
  43                 i915_gem_object_put(obj);
  44
  45         return vma;
  46 }
  47
  48 static struct i915_vma *
  49 hwsp_alloc(struct intel_timeline *timeline, unsigned int *cacheline)
  50 {
  51         struct intel_gt_timelines *gt = &timeline->gt->timelines;
  52         struct intel_timeline_hwsp *hwsp;
  53
  54         BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
  55
  56         spin_lock_irq(&gt->hwsp_lock);
  57
  58         /* hwsp_free_list only contains HWSP that have available cachelines */
  59         hwsp = list_first_entry_or_null(&gt->hwsp_free_list,
  60                                         typeof(*hwsp), free_link);
  61         if (!hwsp) {
  62                 struct i915_vma *vma;
  63
  64                 spin_unlock_irq(&gt->hwsp_lock);
  65
  66                 hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
  67                 if (!hwsp)
  68                         return ERR_PTR(-ENOMEM);
  69
  70                 vma = __hwsp_alloc(timeline->gt);
  71                 if (IS_ERR(vma)) {
  72                         kfree(hwsp);
  73                         return vma;
  74                 }
  75
  76                 GT_TRACE(timeline->gt, "new HWSP allocated\n");
  77
  78                 vma->private = hwsp;
  79                 hwsp->gt = timeline->gt;
  80                 hwsp->vma = vma;
  81                 hwsp->free_bitmap = ~0ull;
  82                 hwsp->gt_timelines = gt;
  83
  84                 spin_lock_irq(&gt->hwsp_lock);
  85                 list_add(&hwsp->free_link, &gt->hwsp_free_list);
  86         }
  87
  88         GEM_BUG_ON(!hwsp->free_bitmap);
  89         *cacheline = __ffs64(hwsp->free_bitmap);
  90         hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
  91         if (!hwsp->free_bitmap)
  92                 list_del(&hwsp->free_link);
  93
  94         spin_unlock_irq(&gt->hwsp_lock);
  95
  96         GEM_BUG_ON(hwsp->vma->private != hwsp);
  97         return hwsp->vma;
  98 }
  99
 100 static void __idle_hwsp_free(struct intel_timeline_hwsp *hwsp, int cacheline)
 101 {
 102         struct intel_gt_timelines *gt = hwsp->gt_timelines;
 103         unsigned long flags;
 104
 105         spin_lock_irqsave(&gt->hwsp_lock, flags);
 106
 107         /* As a cacheline becomes available, publish the HWSP on the freelist */
 108         if (!hwsp->free_bitmap)
 109                 list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);
 110
 111         GEM_BUG_ON(cacheline >= BITS_PER_TYPE(hwsp->free_bitmap));
 112         hwsp->free_bitmap |= BIT_ULL(cacheline);
 113
 114         /* And if no one is left using it, give the page back to the system */
 115         if (hwsp->free_bitmap == ~0ull) {
 116                 i915_vma_put(hwsp->vma);
 117                 list_del(&hwsp->free_link);
 118                 kfree(hwsp);
 119         }
 120
 121         spin_unlock_irqrestore(&gt->hwsp_lock, flags);
 122 }
 123
 124 static void __rcu_cacheline_free(struct rcu_head *rcu)
 125 {
 126         struct intel_timeline_cacheline *cl =
 127                 container_of(rcu, typeof(*cl), rcu);
 128
 129         /* Must wait until after all *rq->hwsp are complete before removing */
 130         i915_gem_object_unpin_map(cl->hwsp->vma->obj);
 131         __idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS));
 132
 133         i915_active_fini(&cl->active);
 134         kfree(cl);
 135 }
 136
 137 static void __idle_cacheline_free(struct intel_timeline_cacheline *cl)
 138 {
 139         GEM_BUG_ON(!i915_active_is_idle(&cl->active));
 140         call_rcu(&cl->rcu, __rcu_cacheline_free);
 141 }
 142
 143 __i915_active_call
 144 static void __cacheline_retire(struct i915_active *active)
 145 {
 146         struct intel_timeline_cacheline *cl =
 147                 container_of(active, typeof(*cl), active);
 148
 149         i915_vma_unpin(cl->hwsp->vma);
 150         if (ptr_test_bit(cl->vaddr, CACHELINE_FREE))
 151                 __idle_cacheline_free(cl);
 152 }
 153
 154 static int __cacheline_active(struct i915_active *active)
 155 {
 156         struct intel_timeline_cacheline *cl =
 157                 container_of(active, typeof(*cl), active);
 158
 159         __i915_vma_pin(cl->hwsp->vma);
 160         return 0;
 161 }
 162
 163 static struct intel_timeline_cacheline *
 164 cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)
 165 {
 166         struct intel_timeline_cacheline *cl;
 167         void *vaddr;
 168
 169         GEM_BUG_ON(cacheline >= BIT(CACHELINE_BITS));
 170
 171         cl = kmalloc(sizeof(*cl), GFP_KERNEL);
 172         if (!cl)
 173                 return ERR_PTR(-ENOMEM);
 174
 175         vaddr = i915_gem_object_pin_map(hwsp->vma->obj, I915_MAP_WB);
 176         if (IS_ERR(vaddr)) {
 177                 kfree(cl);
 178                 return ERR_CAST(vaddr);
 179         }
 180
 181         cl->hwsp = hwsp;
 182         cl->vaddr = page_pack_bits(vaddr, cacheline);
 183
 184         i915_active_init(&cl->active, __cacheline_active, __cacheline_retire);
 185
 186         return cl;
 187 }
 188
 189 static void cacheline_acquire(struct intel_timeline_cacheline *cl,
 190                               u32 ggtt_offset)
 191 {
 192         if (!cl)
 193                 return;
 194
 195         cl->ggtt_offset = ggtt_offset;
 196         i915_active_acquire(&cl->active);
 197 }
 198
 199 static void cacheline_release(struct intel_timeline_cacheline *cl)
 200 {
 201         if (cl)
 202                 i915_active_release(&cl->active);
 203 }
 204
 205 static void cacheline_free(struct intel_timeline_cacheline *cl)
 206 {
 207         if (!i915_active_acquire_if_busy(&cl->active)) {
 208                 __idle_cacheline_free(cl);
 209                 return;
 210         }
 211
 212         GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE));
 213         cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE);
 214
 215         i915_active_release(&cl->active);
 216 }
 217
 218 static int intel_timeline_init(struct intel_timeline *timeline,
 219                                struct intel_gt *gt,
 220                                struct i915_vma *hwsp,
 221                                unsigned int offset)
 222 {
 223         void *vaddr;
 224
 225         kref_init(&timeline->kref);
 226         atomic_set(&timeline->pin_count, 0);
 227
 228         timeline->gt = gt;
 229
 230         timeline->has_initial_breadcrumb = !hwsp;
 231         timeline->hwsp_cacheline = NULL;
 232
 233         if (!hwsp) {
 234                 struct intel_timeline_cacheline *cl;
 235                 unsigned int cacheline;
 236
 237                 hwsp = hwsp_alloc(timeline, &cacheline);
 238                 if (IS_ERR(hwsp))
 239                         return PTR_ERR(hwsp);
 240
 241                 cl = cacheline_alloc(hwsp->private, cacheline);
 242                 if (IS_ERR(cl)) {
 243                         __idle_hwsp_free(hwsp->private, cacheline);
 244                         return PTR_ERR(cl);
 245                 }
 246
 247                 timeline->hwsp_cacheline = cl;
 248                 timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
 249
 250                 vaddr = page_mask_bits(cl->vaddr);
 251         } else {
 252                 timeline->hwsp_offset = offset;
 253                 vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
 254                 if (IS_ERR(vaddr))
 255                         return PTR_ERR(vaddr);
 256         }
 257
 258         timeline->hwsp_seqno =
 259                 memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);
 260
 261         timeline->hwsp_ggtt = i915_vma_get(hwsp);
 262         GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);
 263
 264         timeline->fence_context = dma_fence_context_alloc(1);
 265
 266         mutex_init(&timeline->mutex);
 267
 268         INIT_ACTIVE_FENCE(&timeline->last_request);
 269         INIT_LIST_HEAD(&timeline->requests);
 270
 271         i915_syncmap_init(&timeline->sync);
 272
 273         return 0;
 274 }
 275
 276 void intel_gt_init_timelines(struct intel_gt *gt)
 277 {
 278         struct intel_gt_timelines *timelines = &gt->timelines;
 279
 280         spin_lock_init(&timelines->lock);
 281         INIT_LIST_HEAD(&timelines->active_list);
 282
 283         spin_lock_init(&timelines->hwsp_lock);
 284         INIT_LIST_HEAD(&timelines->hwsp_free_list);
 285 }
 286
 287 static void intel_timeline_fini(struct intel_timeline *timeline)
 288 {
 289         GEM_BUG_ON(atomic_read(&timeline->pin_count));
 290         GEM_BUG_ON(!list_empty(&timeline->requests));
 291         GEM_BUG_ON(timeline->retire);
 292
 293         if (timeline->hwsp_cacheline)
 294                 cacheline_free(timeline->hwsp_cacheline);
 295         else
 296                 i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
 297
 298         i915_vma_put(timeline->hwsp_ggtt);
 299
 300         /*
 301          * A small race exists between intel_gt_retire_requests_timeout and
 302          * intel_timeline_exit which could result in the syncmap not getting
 303          * free'd. Rather than work to hard to seal this race, simply cleanup
 304          * the syncmap on fini.
 305          */
 306         i915_syncmap_free(&timeline->sync);
 307 }
 308
 309 struct intel_timeline *
 310 __intel_timeline_create(struct intel_gt *gt,
 311                         struct i915_vma *global_hwsp,
 312                         unsigned int offset)
 313 {
 314         struct intel_timeline *timeline;
 315         int err;
 316
 317         timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
 318         if (!timeline)
 319                 return ERR_PTR(-ENOMEM);
 320
 321         err = intel_timeline_init(timeline, gt, global_hwsp, offset);
 322         if (err) {
 323                 kfree(timeline);
 324                 return ERR_PTR(err);
 325         }
 326
 327         return timeline;
 328 }
 329
 330 void __intel_timeline_pin(struct intel_timeline *tl)
 331 {
 332         GEM_BUG_ON(!atomic_read(&tl->pin_count));
 333         atomic_inc(&tl->pin_count);
 334 }
 335
 336 int intel_timeline_pin(struct intel_timeline *tl, struct i915_gem_ww_ctx *ww)
 337 {
 338         int err;
 339
 340         if (atomic_add_unless(&tl->pin_count, 1, 0))
 341                 return 0;
 342
 343         err = i915_ggtt_pin(tl->hwsp_ggtt, ww, 0, PIN_HIGH);
 344         if (err)
 345                 return err;
 346
 347         tl->hwsp_offset =
 348                 i915_ggtt_offset(tl->hwsp_ggtt) +
 349                 offset_in_page(tl->hwsp_offset);
 350         GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
 351                  tl->fence_context, tl->hwsp_offset);
 352
 353         cacheline_acquire(tl->hwsp_cacheline, tl->hwsp_offset);
 354         if (atomic_fetch_inc(&tl->pin_count)) {
 355                 cacheline_release(tl->hwsp_cacheline);
 356                 __i915_vma_unpin(tl->hwsp_ggtt);
 357         }
 358
 359         return 0;
 360 }
 361
 362 void intel_timeline_reset_seqno(const struct intel_timeline *tl)
 363 {
 364         /* Must be pinned to be writable, and no requests in flight. */
 365         GEM_BUG_ON(!atomic_read(&tl->pin_count));
 366         WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);
 367 }
 368
 369 void intel_timeline_enter(struct intel_timeline *tl)
 370 {
 371         struct intel_gt_timelines *timelines = &tl->gt->timelines;
 372
 373         /*
 374          * Pretend we are serialised by the timeline->mutex.
 375          *
 376          * While generally true, there are a few exceptions to the rule
 377          * for the engine->kernel_context being used to manage power
 378          * transitions. As the engine_park may be called from under any
 379          * timeline, it uses the power mutex as a global serialisation
 380          * lock to prevent any other request entering its timeline.
 381          *
 382          * The rule is generally tl->mutex, otherwise engine->wakeref.mutex.
 383          *
 384          * However, intel_gt_retire_request() does not know which engine
 385          * it is retiring along and so cannot partake in the engine-pm
 386          * barrier, and there we use the tl->active_count as a means to
 387          * pin the timeline in the active_list while the locks are dropped.
 388          * Ergo, as that is outside of the engine-pm barrier, we need to
 389          * use atomic to manipulate tl->active_count.
 390          */
 391         lockdep_assert_held(&tl->mutex);
 392
 393         if (atomic_add_unless(&tl->active_count, 1, 0))
 394                 return;
 395
 396         spin_lock(&timelines->lock);
 397         if (!atomic_fetch_inc(&tl->active_count)) {
 398                 /*
 399                  * The HWSP is volatile, and may have been lost while inactive,
 400                  * e.g. across suspend/resume. Be paranoid, and ensure that
 401                  * the HWSP value matches our seqno so we don't proclaim
 402                  * the next request as already complete.
 403                  */
 404                 intel_timeline_reset_seqno(tl);
 405                 list_add_tail(&tl->link, &timelines->active_list);
 406         }
 407         spin_unlock(&timelines->lock);
 408 }
 409
 410 void intel_timeline_exit(struct intel_timeline *tl)
 411 {
 412         struct intel_gt_timelines *timelines = &tl->gt->timelines;
 413
 414         /* See intel_timeline_enter() */
 415         lockdep_assert_held(&tl->mutex);
 416
 417         GEM_BUG_ON(!atomic_read(&tl->active_count));
 418         if (atomic_add_unless(&tl->active_count, -1, 1))
 419                 return;
 420
 421         spin_lock(&timelines->lock);
 422         if (atomic_dec_and_test(&tl->active_count))
 423                 list_del(&tl->link);
 424         spin_unlock(&timelines->lock);
 425
 426         /*
 427          * Since this timeline is idle, all bariers upon which we were waiting
 428          * must also be complete and so we can discard the last used barriers
 429          * without loss of information.
 430          */
 431         i915_syncmap_free(&tl->sync);
 432 }
 433
 434 static u32 timeline_advance(struct intel_timeline *tl)
 435 {
 436         GEM_BUG_ON(!atomic_read(&tl->pin_count));
 437         GEM_BUG_ON(tl->seqno & tl->has_initial_breadcrumb);
 438
 439         return tl->seqno += 1 + tl->has_initial_breadcrumb;
 440 }
 441
 442 static void timeline_rollback(struct intel_timeline *tl)
 443 {
 444         tl->seqno -= 1 + tl->has_initial_breadcrumb;
 445 }
 446
 447 static noinline int
 448 __intel_timeline_get_seqno(struct intel_timeline *tl,
 449                            struct i915_request *rq,
 450                            u32 *seqno)
 451 {
 452         struct intel_timeline_cacheline *cl;
 453         unsigned int cacheline;
 454         struct i915_vma *vma;
 455         void *vaddr;
 456         int err;
 457
 458         might_lock(&tl->gt->ggtt->vm.mutex);
 459         GT_TRACE(tl->gt, "timeline:%llx wrapped\n", tl->fence_context);
 460
 461         /*
 462          * If there is an outstanding GPU reference to this cacheline,
 463          * such as it being sampled by a HW semaphore on another timeline,
 464          * we cannot wraparound our seqno value (the HW semaphore does
 465          * a strict greater-than-or-equals compare, not i915_seqno_passed).
 466          * So if the cacheline is still busy, we must detach ourselves
 467          * from it and leave it inflight alongside its users.
 468          *
 469          * However, if nobody is watching and we can guarantee that nobody
 470          * will, we could simply reuse the same cacheline.
 471          *
 472          * if (i915_active_request_is_signaled(&tl->last_request) &&
 473          *     i915_active_is_signaled(&tl->hwsp_cacheline->active))
 474          *      return 0;
 475          *
 476          * That seems unlikely for a busy timeline that needed to wrap in
 477          * the first place, so just replace the cacheline.
 478          */
 479
 480         vma = hwsp_alloc(tl, &cacheline);
 481         if (IS_ERR(vma)) {
 482                 err = PTR_ERR(vma);
 483                 goto err_rollback;
 484         }
 485
 486         err = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
 487         if (err) {
 488                 __idle_hwsp_free(vma->private, cacheline);
 489                 goto err_rollback;
 490         }
 491
 492         cl = cacheline_alloc(vma->private, cacheline);
 493         if (IS_ERR(cl)) {
 494                 err = PTR_ERR(cl);
 495                 __idle_hwsp_free(vma->private, cacheline);
 496                 goto err_unpin;
 497         }
 498         GEM_BUG_ON(cl->hwsp->vma != vma);
 499
 500         /*
 501          * Attach the old cacheline to the current request, so that we only
 502          * free it after the current request is retired, which ensures that
 503          * all writes into the cacheline from previous requests are complete.
 504          */
 505         err = i915_active_ref(&tl->hwsp_cacheline->active,
 506                               tl->fence_context,
 507                               &rq->fence);
 508         if (err)
 509                 goto err_cacheline;
 510
 511         cacheline_release(tl->hwsp_cacheline); /* ownership now xfered to rq */
 512         cacheline_free(tl->hwsp_cacheline);
 513
 514         i915_vma_unpin(tl->hwsp_ggtt); /* binding kept alive by old cacheline */
 515         i915_vma_put(tl->hwsp_ggtt);
 516
 517         tl->hwsp_ggtt = i915_vma_get(vma);
 518
 519         vaddr = page_mask_bits(cl->vaddr);
 520         tl->hwsp_offset = cacheline * CACHELINE_BYTES;
 521         tl->hwsp_seqno =
 522                 memset(vaddr + tl->hwsp_offset, 0, CACHELINE_BYTES);
 523
 524         tl->hwsp_offset += i915_ggtt_offset(vma);
 525         GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
 526                  tl->fence_context, tl->hwsp_offset);
 527
 528         cacheline_acquire(cl, tl->hwsp_offset);
 529         tl->hwsp_cacheline = cl;
 530
 531         *seqno = timeline_advance(tl);
 532         GEM_BUG_ON(i915_seqno_passed(*tl->hwsp_seqno, *seqno));
 533         return 0;
 534
 535 err_cacheline:
 536         cacheline_free(cl);
 537 err_unpin:
 538         i915_vma_unpin(vma);
 539 err_rollback:
 540         timeline_rollback(tl);
 541         return err;
 542 }
 543
 544 int intel_timeline_get_seqno(struct intel_timeline *tl,
 545                              struct i915_request *rq,
 546                              u32 *seqno)
 547 {
 548         *seqno = timeline_advance(tl);
 549
 550         /* Replace the HWSP on wraparound for HW semaphores */
 551         if (unlikely(!*seqno && tl->hwsp_cacheline))
 552                 return __intel_timeline_get_seqno(tl, rq, seqno);
 553
 554         return 0;
 555 }
 556
 557 static int cacheline_ref(struct intel_timeline_cacheline *cl,
 558                          struct i915_request *rq)
 559 {
 560         return i915_active_add_request(&cl->active, rq);
 561 }
 562
 563 int intel_timeline_read_hwsp(struct i915_request *from,
 564                              struct i915_request *to,
 565                              u32 *hwsp)
 566 {
 567         struct intel_timeline_cacheline *cl;
 568         int err;
 569
 570         GEM_BUG_ON(!rcu_access_pointer(from->hwsp_cacheline));
 571
 572         rcu_read_lock();
 573         cl = rcu_dereference(from->hwsp_cacheline);
 574         if (i915_request_completed(from)) /* confirm cacheline is valid */
 575                 goto unlock;
 576         if (unlikely(!i915_active_acquire_if_busy(&cl->active)))
 577                 goto unlock; /* seqno wrapped and completed! */
 578         if (unlikely(i915_request_completed(from)))
 579                 goto release;
 580         rcu_read_unlock();
 581
 582         err = cacheline_ref(cl, to);
 583         if (err)
 584                 goto out;
 585
 586         *hwsp = cl->ggtt_offset;
 587 out:
 588         i915_active_release(&cl->active);
 589         return err;
 590
 591 release:
 592         i915_active_release(&cl->active);
 593 unlock:
 594         rcu_read_unlock();
 595         return 1;
 596 }
 597
 598 void intel_timeline_unpin(struct intel_timeline *tl)
 599 {
 600         GEM_BUG_ON(!atomic_read(&tl->pin_count));
 601         if (!atomic_dec_and_test(&tl->pin_count))
 602                 return;
 603
 604         cacheline_release(tl->hwsp_cacheline);
 605
 606         __i915_vma_unpin(tl->hwsp_ggtt);
 607 }
 608
 609 void __intel_timeline_free(struct kref *kref)
 610 {
 611         struct intel_timeline *timeline =
 612                 container_of(kref, typeof(*timeline), kref);
 613
 614         intel_timeline_fini(timeline);
 615         kfree_rcu(timeline, rcu);
 616 }
 617
 618 void intel_gt_fini_timelines(struct intel_gt *gt)
 619 {
 620         struct intel_gt_timelines *timelines = &gt->timelines;
 621
 622         GEM_BUG_ON(!list_empty(&timelines->active_list));
 623         GEM_BUG_ON(!list_empty(&timelines->hwsp_free_list));
 624 }
 625
 626 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 627 #include "gt/selftests/mock_timeline.c"
 628 #include "gt/selftest_timeline.c"
 629 #endif