drivers/gpu/drm/i915/i915_sw_fence.c

   1 /*
   2  * (C) Copyright 2016 Intel Corporation
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License
   6  * as published by the Free Software Foundation; version 2
   7  * of the License.
   8  */
   9
  10 #include <linux/slab.h>
  11 #include <linux/dma-fence.h>
  12 #include <linux/reservation.h>
  13
  14 #include "i915_sw_fence.h"
  15 #include "i915_selftest.h"
  16
  17 #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
  18
  19 static DEFINE_SPINLOCK(i915_sw_fence_lock);
  20
  21 enum {
  22         DEBUG_FENCE_IDLE = 0,
  23         DEBUG_FENCE_NOTIFY,
  24 };
  25
  26 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
  27
  28 static void *i915_sw_fence_debug_hint(void *addr)
  29 {
  30         return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
  31 }
  32
  33 static struct debug_obj_descr i915_sw_fence_debug_descr = {
  34         .name = "i915_sw_fence",
  35         .debug_hint = i915_sw_fence_debug_hint,
  36 };
  37
  38 static inline void debug_fence_init(struct i915_sw_fence *fence)
  39 {
  40         debug_object_init(fence, &i915_sw_fence_debug_descr);
  41 }
  42
  43 static inline void debug_fence_activate(struct i915_sw_fence *fence)
  44 {
  45         debug_object_activate(fence, &i915_sw_fence_debug_descr);
  46 }
  47
  48 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
  49                                          int old, int new)
  50 {
  51         debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
  52 }
  53
  54 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
  55 {
  56         debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
  57 }
  58
  59 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
  60 {
  61         debug_object_destroy(fence, &i915_sw_fence_debug_descr);
  62 }
  63
  64 static inline void debug_fence_free(struct i915_sw_fence *fence)
  65 {
  66         debug_object_free(fence, &i915_sw_fence_debug_descr);
  67         smp_wmb(); /* flush the change in state before reallocation */
  68 }
  69
  70 static inline void debug_fence_assert(struct i915_sw_fence *fence)
  71 {
  72         debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
  73 }
  74
  75 #else
  76
  77 static inline void debug_fence_init(struct i915_sw_fence *fence)
  78 {
  79 }
  80
  81 static inline void debug_fence_activate(struct i915_sw_fence *fence)
  82 {
  83 }
  84
  85 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
  86                                          int old, int new)
  87 {
  88 }
  89
  90 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
  91 {
  92 }
  93
  94 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
  95 {
  96 }
  97
  98 static inline void debug_fence_free(struct i915_sw_fence *fence)
  99 {
 100 }
 101
 102 static inline void debug_fence_assert(struct i915_sw_fence *fence)
 103 {
 104 }
 105
 106 #endif
 107
 108 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
 109                                   enum i915_sw_fence_notify state)
 110 {
 111         i915_sw_fence_notify_t fn;
 112
 113         fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
 114         return fn(fence, state);
 115 }
 116
 117 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
 118 void i915_sw_fence_fini(struct i915_sw_fence *fence)
 119 {
 120         debug_fence_free(fence);
 121 }
 122 #endif
 123
 124 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
 125                                         struct list_head *continuation)
 126 {
 127         wait_queue_head_t *x = &fence->wait;
 128         wait_queue_entry_t *pos, *next;
 129         unsigned long flags;
 130
 131         debug_fence_deactivate(fence);
 132         atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
 133
 134         /*
 135          * To prevent unbounded recursion as we traverse the graph of
 136          * i915_sw_fences, we move the entry list from this, the next ready
 137          * fence, to the tail of the original fence's entry list
 138          * (and so added to the list to be woken).
 139          */
 140
 141         spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
 142         if (continuation) {
 143                 list_for_each_entry_safe(pos, next, &x->head, entry) {
 144                         if (pos->func == autoremove_wake_function)
 145                                 pos->func(pos, TASK_NORMAL, 0, continuation);
 146                         else
 147                                 list_move_tail(&pos->entry, continuation);
 148                 }
 149         } else {
 150                 LIST_HEAD(extra);
 151
 152                 do {
 153                         list_for_each_entry_safe(pos, next, &x->head, entry)
 154                                 pos->func(pos, TASK_NORMAL, 0, &extra);
 155
 156                         if (list_empty(&extra))
 157                                 break;
 158
 159                         list_splice_tail_init(&extra, &x->head);
 160                 } while (1);
 161         }
 162         spin_unlock_irqrestore(&x->lock, flags);
 163
 164         debug_fence_assert(fence);
 165 }
 166
 167 static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
 168                                      struct list_head *continuation)
 169 {
 170         debug_fence_assert(fence);
 171
 172         if (!atomic_dec_and_test(&fence->pending))
 173                 return;
 174
 175         debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
 176
 177         if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
 178                 return;
 179
 180         debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
 181
 182         __i915_sw_fence_wake_up_all(fence, continuation);
 183
 184         debug_fence_destroy(fence);
 185         __i915_sw_fence_notify(fence, FENCE_FREE);
 186 }
 187
 188 static void i915_sw_fence_complete(struct i915_sw_fence *fence)
 189 {
 190         debug_fence_assert(fence);
 191
 192         if (WARN_ON(i915_sw_fence_done(fence)))
 193                 return;
 194
 195         __i915_sw_fence_complete(fence, NULL);
 196 }
 197
 198 static void i915_sw_fence_await(struct i915_sw_fence *fence)
 199 {
 200         debug_fence_assert(fence);
 201         WARN_ON(atomic_inc_return(&fence->pending) <= 1);
 202 }
 203
 204 void __i915_sw_fence_init(struct i915_sw_fence *fence,
 205                           i915_sw_fence_notify_t fn,
 206                           const char *name,
 207                           struct lock_class_key *key)
 208 {
 209         BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK);
 210
 211         debug_fence_init(fence);
 212
 213         __init_waitqueue_head(&fence->wait, name, key);
 214         atomic_set(&fence->pending, 1);
 215         fence->flags = (unsigned long)fn;
 216 }
 217
 218 void i915_sw_fence_commit(struct i915_sw_fence *fence)
 219 {
 220         debug_fence_activate(fence);
 221         i915_sw_fence_complete(fence);
 222 }
 223
 224 static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
 225 {
 226         list_del(&wq->entry);
 227         __i915_sw_fence_complete(wq->private, key);
 228
 229         if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
 230                 kfree(wq);
 231         return 0;
 232 }
 233
 234 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 235                                     const struct i915_sw_fence * const signaler)
 236 {
 237         wait_queue_entry_t *wq;
 238
 239         if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 240                 return false;
 241
 242         if (fence == signaler)
 243                 return true;
 244
 245         list_for_each_entry(wq, &fence->wait.head, entry) {
 246                 if (wq->func != i915_sw_fence_wake)
 247                         continue;
 248
 249                 if (__i915_sw_fence_check_if_after(wq->private, signaler))
 250                         return true;
 251         }
 252
 253         return false;
 254 }
 255
 256 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
 257 {
 258         wait_queue_entry_t *wq;
 259
 260         if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 261                 return;
 262
 263         list_for_each_entry(wq, &fence->wait.head, entry) {
 264                 if (wq->func != i915_sw_fence_wake)
 265                         continue;
 266
 267                 __i915_sw_fence_clear_checked_bit(wq->private);
 268         }
 269 }
 270
 271 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 272                                   const struct i915_sw_fence * const signaler)
 273 {
 274         unsigned long flags;
 275         bool err;
 276
 277         if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
 278                 return false;
 279
 280         spin_lock_irqsave(&i915_sw_fence_lock, flags);
 281         err = __i915_sw_fence_check_if_after(fence, signaler);
 282         __i915_sw_fence_clear_checked_bit(fence);
 283         spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
 284
 285         return err;
 286 }
 287
 288 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 289                                           struct i915_sw_fence *signaler,
 290                                           wait_queue_entry_t *wq, gfp_t gfp)
 291 {
 292         unsigned long flags;
 293         int pending;
 294
 295         debug_fence_assert(fence);
 296
 297         if (i915_sw_fence_done(signaler))
 298                 return 0;
 299
 300         debug_fence_assert(signaler);
 301
 302         /* The dependency graph must be acyclic. */
 303         if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
 304                 return -EINVAL;
 305
 306         pending = 0;
 307         if (!wq) {
 308                 wq = kmalloc(sizeof(*wq), gfp);
 309                 if (!wq) {
 310                         if (!gfpflags_allow_blocking(gfp))
 311                                 return -ENOMEM;
 312
 313                         i915_sw_fence_wait(signaler);
 314                         return 0;
 315                 }
 316
 317                 pending |= I915_SW_FENCE_FLAG_ALLOC;
 318         }
 319
 320         INIT_LIST_HEAD(&wq->entry);
 321         wq->flags = pending;
 322         wq->func = i915_sw_fence_wake;
 323         wq->private = fence;
 324
 325         i915_sw_fence_await(fence);
 326
 327         spin_lock_irqsave(&signaler->wait.lock, flags);
 328         if (likely(!i915_sw_fence_done(signaler))) {
 329                 __add_wait_queue_entry_tail(&signaler->wait, wq);
 330                 pending = 1;
 331         } else {
 332                 i915_sw_fence_wake(wq, 0, 0, NULL);
 333                 pending = 0;
 334         }
 335         spin_unlock_irqrestore(&signaler->wait.lock, flags);
 336
 337         return pending;
 338 }
 339
 340 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 341                                  struct i915_sw_fence *signaler,
 342                                  wait_queue_entry_t *wq)
 343 {
 344         return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
 345 }
 346
 347 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
 348                                      struct i915_sw_fence *signaler,
 349                                      gfp_t gfp)
 350 {
 351         return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
 352 }
 353
 354 struct i915_sw_dma_fence_cb {
 355         struct dma_fence_cb base;
 356         struct i915_sw_fence *fence;
 357         struct dma_fence *dma;
 358         struct timer_list timer;
 359 };
 360
 361 static void timer_i915_sw_fence_wake(unsigned long data)
 362 {
 363         struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data;
 364
 365         pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
 366                 cb->dma->ops->get_driver_name(cb->dma),
 367                 cb->dma->ops->get_timeline_name(cb->dma),
 368                 cb->dma->seqno);
 369         dma_fence_put(cb->dma);
 370         cb->dma = NULL;
 371
 372         i915_sw_fence_complete(cb->fence);
 373         cb->timer.function = NULL;
 374 }
 375
 376 static void dma_i915_sw_fence_wake(struct dma_fence *dma,
 377                                    struct dma_fence_cb *data)
 378 {
 379         struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
 380
 381         del_timer_sync(&cb->timer);
 382         if (cb->timer.function)
 383                 i915_sw_fence_complete(cb->fence);
 384         dma_fence_put(cb->dma);
 385
 386         kfree(cb);
 387 }
 388
 389 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 390                                   struct dma_fence *dma,
 391                                   unsigned long timeout,
 392                                   gfp_t gfp)
 393 {
 394         struct i915_sw_dma_fence_cb *cb;
 395         int ret;
 396
 397         debug_fence_assert(fence);
 398
 399         if (dma_fence_is_signaled(dma))
 400                 return 0;
 401
 402         cb = kmalloc(sizeof(*cb), gfp);
 403         if (!cb) {
 404                 if (!gfpflags_allow_blocking(gfp))
 405                         return -ENOMEM;
 406
 407                 return dma_fence_wait(dma, false);
 408         }
 409
 410         cb->fence = fence;
 411         i915_sw_fence_await(fence);
 412
 413         cb->dma = NULL;
 414         __setup_timer(&cb->timer,
 415                       timer_i915_sw_fence_wake, (unsigned long)cb,
 416                       TIMER_IRQSAFE);
 417         if (timeout) {
 418                 cb->dma = dma_fence_get(dma);
 419                 mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
 420         }
 421
 422         ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
 423         if (ret == 0) {
 424                 ret = 1;
 425         } else {
 426                 dma_i915_sw_fence_wake(dma, &cb->base);
 427                 if (ret == -ENOENT) /* fence already signaled */
 428                         ret = 0;
 429         }
 430
 431         return ret;
 432 }
 433
 434 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 435                                     struct reservation_object *resv,
 436                                     const struct dma_fence_ops *exclude,
 437                                     bool write,
 438                                     unsigned long timeout,
 439                                     gfp_t gfp)
 440 {
 441         struct dma_fence *excl;
 442         int ret = 0, pending;
 443
 444         debug_fence_assert(fence);
 445
 446         if (write) {
 447                 struct dma_fence **shared;
 448                 unsigned int count, i;
 449
 450                 ret = reservation_object_get_fences_rcu(resv,
 451                                                         &excl, &count, &shared);
 452                 if (ret)
 453                         return ret;
 454
 455                 for (i = 0; i < count; i++) {
 456                         if (shared[i]->ops == exclude)
 457                                 continue;
 458
 459                         pending = i915_sw_fence_await_dma_fence(fence,
 460                                                                 shared[i],
 461                                                                 timeout,
 462                                                                 gfp);
 463                         if (pending < 0) {
 464                                 ret = pending;
 465                                 break;
 466                         }
 467
 468                         ret |= pending;
 469                 }
 470
 471                 for (i = 0; i < count; i++)
 472                         dma_fence_put(shared[i]);
 473                 kfree(shared);
 474         } else {
 475                 excl = reservation_object_get_excl_rcu(resv);
 476         }
 477
 478         if (ret >= 0 && excl && excl->ops != exclude) {
 479                 pending = i915_sw_fence_await_dma_fence(fence,
 480                                                         excl,
 481                                                         timeout,
 482                                                         gfp);
 483                 if (pending < 0)
 484                         ret = pending;
 485                 else
 486                         ret |= pending;
 487         }
 488
 489         dma_fence_put(excl);
 490
 491         return ret;
 492 }
 493
 494 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 495 #include "selftests/i915_sw_fence.c"
 496 #endif