2 * Copyright 2017 Red Hat
3 * Parts ported from amdgpu (fence wait code).
4 * Copyright 2016 Advanced Micro Devices, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
32 * DRM synchronisation objects (syncobj) are a persistent objects,
33 * that contain an optional fence. The fence can be updated with a new
36 * syncobj's can be waited upon, where it will wait for the underlying
39 * syncobj's can be export to fd's and back, these fd's are opaque and
40 * have no other use case, except passing the syncobj between processes.
42 * Their primary use-case is to implement Vulkan fences and semaphores.
44 * syncobj have a kref reference count, but also have an optional file.
45 * The file is only created once the syncobj is exported.
46 * The file takes a reference on the kref.
50 #include <linux/file.h>
52 #include <linux/anon_inodes.h>
53 #include <linux/sync_file.h>
54 #include <linux/sched/signal.h>
56 #include "drm_internal.h"
57 #include <drm/drm_syncobj.h>
60 * drm_syncobj_find - lookup and reference a sync object.
61 * @file_private: drm file private pointer
62 * @handle: sync object handle to lookup.
64 * Returns a reference to the syncobj pointed to by handle or NULL.
66 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
69 struct drm_syncobj *syncobj;
71 spin_lock(&file_private->syncobj_table_lock);
73 /* Check if we currently have a reference on the object */
74 syncobj = idr_find(&file_private->syncobj_idr, handle);
76 drm_syncobj_get(syncobj);
78 spin_unlock(&file_private->syncobj_table_lock);
82 EXPORT_SYMBOL(drm_syncobj_find);
84 static void drm_syncobj_add_callback_locked(struct drm_syncobj *syncobj,
85 struct drm_syncobj_cb *cb,
86 drm_syncobj_func_t func)
89 list_add_tail(&cb->node, &syncobj->cb_list);
92 static int drm_syncobj_fence_get_or_add_callback(struct drm_syncobj *syncobj,
93 struct dma_fence **fence,
94 struct drm_syncobj_cb *cb,
95 drm_syncobj_func_t func)
101 *fence = drm_syncobj_fence_get(syncobj);
105 spin_lock(&syncobj->lock);
106 /* We've already tried once to get a fence and failed. Now that we
107 * have the lock, try one more time just to be sure we don't add a
108 * callback when a fence has already been set.
110 if (syncobj->fence) {
111 *fence = dma_fence_get(syncobj->fence);
115 drm_syncobj_add_callback_locked(syncobj, cb, func);
118 spin_unlock(&syncobj->lock);
124 * drm_syncobj_add_callback - adds a callback to syncobj::cb_list
125 * @syncobj: Sync object to which to add the callback
126 * @cb: Callback to add
127 * @func: Func to use when initializing the drm_syncobj_cb struct
129 * This adds a callback to be called next time the fence is replaced
131 void drm_syncobj_add_callback(struct drm_syncobj *syncobj,
132 struct drm_syncobj_cb *cb,
133 drm_syncobj_func_t func)
135 spin_lock(&syncobj->lock);
136 drm_syncobj_add_callback_locked(syncobj, cb, func);
137 spin_unlock(&syncobj->lock);
139 EXPORT_SYMBOL(drm_syncobj_add_callback);
142 * drm_syncobj_add_callback - removes a callback to syncobj::cb_list
143 * @syncobj: Sync object from which to remove the callback
144 * @cb: Callback to remove
146 void drm_syncobj_remove_callback(struct drm_syncobj *syncobj,
147 struct drm_syncobj_cb *cb)
149 spin_lock(&syncobj->lock);
150 list_del_init(&cb->node);
151 spin_unlock(&syncobj->lock);
153 EXPORT_SYMBOL(drm_syncobj_remove_callback);
156 * drm_syncobj_replace_fence - replace fence in a sync object.
157 * @syncobj: Sync object to replace fence in
158 * @fence: fence to install in sync file.
160 * This replaces the fence on a sync object.
162 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
163 struct dma_fence *fence)
165 struct dma_fence *old_fence;
166 struct drm_syncobj_cb *cur, *tmp;
169 dma_fence_get(fence);
171 spin_lock(&syncobj->lock);
173 old_fence = syncobj->fence;
174 syncobj->fence = fence;
176 if (fence != old_fence) {
177 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
178 list_del_init(&cur->node);
179 cur->func(syncobj, cur);
183 spin_unlock(&syncobj->lock);
185 dma_fence_put(old_fence);
187 EXPORT_SYMBOL(drm_syncobj_replace_fence);
189 struct drm_syncobj_null_fence {
190 struct dma_fence base;
194 static const char *drm_syncobj_null_fence_get_name(struct dma_fence *fence)
196 return "syncobjnull";
199 static bool drm_syncobj_null_fence_enable_signaling(struct dma_fence *fence)
201 dma_fence_enable_sw_signaling(fence);
202 return !dma_fence_is_signaled(fence);
205 static const struct dma_fence_ops drm_syncobj_null_fence_ops = {
206 .get_driver_name = drm_syncobj_null_fence_get_name,
207 .get_timeline_name = drm_syncobj_null_fence_get_name,
208 .enable_signaling = drm_syncobj_null_fence_enable_signaling,
209 .wait = dma_fence_default_wait,
213 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
215 struct drm_syncobj_null_fence *fence;
216 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
220 spin_lock_init(&fence->lock);
221 dma_fence_init(&fence->base, &drm_syncobj_null_fence_ops,
223 dma_fence_signal(&fence->base);
225 drm_syncobj_replace_fence(syncobj, &fence->base);
227 dma_fence_put(&fence->base);
232 int drm_syncobj_find_fence(struct drm_file *file_private,
234 struct dma_fence **fence)
236 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
242 *fence = drm_syncobj_fence_get(syncobj);
246 drm_syncobj_put(syncobj);
249 EXPORT_SYMBOL(drm_syncobj_find_fence);
252 * drm_syncobj_free - free a sync object.
253 * @kref: kref to free.
255 * Only to be called from kref_put in drm_syncobj_put.
257 void drm_syncobj_free(struct kref *kref)
259 struct drm_syncobj *syncobj = container_of(kref,
262 drm_syncobj_replace_fence(syncobj, NULL);
265 EXPORT_SYMBOL(drm_syncobj_free);
267 static int drm_syncobj_create(struct drm_file *file_private,
268 u32 *handle, uint32_t flags)
271 struct drm_syncobj *syncobj;
273 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
277 kref_init(&syncobj->refcount);
278 INIT_LIST_HEAD(&syncobj->cb_list);
279 spin_lock_init(&syncobj->lock);
281 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
282 ret = drm_syncobj_assign_null_handle(syncobj);
284 drm_syncobj_put(syncobj);
289 idr_preload(GFP_KERNEL);
290 spin_lock(&file_private->syncobj_table_lock);
291 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
292 spin_unlock(&file_private->syncobj_table_lock);
297 drm_syncobj_put(syncobj);
305 static int drm_syncobj_destroy(struct drm_file *file_private,
308 struct drm_syncobj *syncobj;
310 spin_lock(&file_private->syncobj_table_lock);
311 syncobj = idr_remove(&file_private->syncobj_idr, handle);
312 spin_unlock(&file_private->syncobj_table_lock);
317 drm_syncobj_put(syncobj);
321 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
323 struct drm_syncobj *syncobj = file->private_data;
325 drm_syncobj_put(syncobj);
329 static const struct file_operations drm_syncobj_file_fops = {
330 .release = drm_syncobj_file_release,
333 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
334 u32 handle, int *p_fd)
336 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
343 fd = get_unused_fd_flags(O_CLOEXEC);
345 drm_syncobj_put(syncobj);
349 file = anon_inode_getfile("syncobj_file",
350 &drm_syncobj_file_fops,
354 drm_syncobj_put(syncobj);
355 return PTR_ERR(file);
358 fd_install(fd, file);
364 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
367 struct drm_syncobj *syncobj;
375 if (file->f_op != &drm_syncobj_file_fops) {
380 /* take a reference to put in the idr */
381 syncobj = file->private_data;
382 drm_syncobj_get(syncobj);
384 idr_preload(GFP_KERNEL);
385 spin_lock(&file_private->syncobj_table_lock);
386 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
387 spin_unlock(&file_private->syncobj_table_lock);
394 drm_syncobj_put(syncobj);
400 int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
403 struct dma_fence *fence = sync_file_get_fence(fd);
404 struct drm_syncobj *syncobj;
409 syncobj = drm_syncobj_find(file_private, handle);
411 dma_fence_put(fence);
415 drm_syncobj_replace_fence(syncobj, fence);
416 dma_fence_put(fence);
417 drm_syncobj_put(syncobj);
421 int drm_syncobj_export_sync_file(struct drm_file *file_private,
422 int handle, int *p_fd)
425 struct dma_fence *fence;
426 struct sync_file *sync_file;
427 int fd = get_unused_fd_flags(O_CLOEXEC);
432 ret = drm_syncobj_find_fence(file_private, handle, &fence);
436 sync_file = sync_file_create(fence);
438 dma_fence_put(fence);
445 fd_install(fd, sync_file->file);
454 * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
455 * @file_private: drm file-private structure to set up
457 * Called at device open time, sets up the structure for handling refcounting
461 drm_syncobj_open(struct drm_file *file_private)
463 idr_init(&file_private->syncobj_idr);
464 spin_lock_init(&file_private->syncobj_table_lock);
468 drm_syncobj_release_handle(int id, void *ptr, void *data)
470 struct drm_syncobj *syncobj = ptr;
472 drm_syncobj_put(syncobj);
477 * drm_syncobj_release - release file-private sync object resources
478 * @file_private: drm file-private structure to clean up
480 * Called at close time when the filp is going away.
482 * Releases any remaining references on objects by this filp.
485 drm_syncobj_release(struct drm_file *file_private)
487 idr_for_each(&file_private->syncobj_idr,
488 &drm_syncobj_release_handle, file_private);
489 idr_destroy(&file_private->syncobj_idr);
493 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
494 struct drm_file *file_private)
496 struct drm_syncobj_create *args = data;
498 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
501 /* no valid flags yet */
502 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
505 return drm_syncobj_create(file_private,
506 &args->handle, args->flags);
510 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
511 struct drm_file *file_private)
513 struct drm_syncobj_destroy *args = data;
515 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
518 /* make sure padding is empty */
521 return drm_syncobj_destroy(file_private, args->handle);
525 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
526 struct drm_file *file_private)
528 struct drm_syncobj_handle *args = data;
530 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
536 if (args->flags != 0 &&
537 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
540 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
541 return drm_syncobj_export_sync_file(file_private, args->handle,
544 return drm_syncobj_handle_to_fd(file_private, args->handle,
549 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
550 struct drm_file *file_private)
552 struct drm_syncobj_handle *args = data;
554 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
560 if (args->flags != 0 &&
561 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
564 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
565 return drm_syncobj_import_sync_file_fence(file_private,
569 return drm_syncobj_fd_to_handle(file_private, args->fd,
573 struct syncobj_wait_entry {
574 struct task_struct *task;
575 struct dma_fence *fence;
576 struct dma_fence_cb fence_cb;
577 struct drm_syncobj_cb syncobj_cb;
580 static void syncobj_wait_fence_func(struct dma_fence *fence,
581 struct dma_fence_cb *cb)
583 struct syncobj_wait_entry *wait =
584 container_of(cb, struct syncobj_wait_entry, fence_cb);
586 wake_up_process(wait->task);
589 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
590 struct drm_syncobj_cb *cb)
592 struct syncobj_wait_entry *wait =
593 container_of(cb, struct syncobj_wait_entry, syncobj_cb);
595 /* This happens inside the syncobj lock */
596 wait->fence = dma_fence_get(syncobj->fence);
597 wake_up_process(wait->task);
600 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
606 struct syncobj_wait_entry *entries;
607 struct dma_fence *fence;
609 uint32_t signaled_count, i;
611 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
615 /* Walk the list of sync objects and initialize entries. We do
616 * this up-front so that we can properly return -EINVAL if there is
617 * a syncobj with a missing fence and then never have the chance of
618 * returning -EINVAL again.
621 for (i = 0; i < count; ++i) {
622 entries[i].task = current;
623 entries[i].fence = drm_syncobj_fence_get(syncobjs[i]);
624 if (!entries[i].fence) {
625 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
629 goto cleanup_entries;
633 if (dma_fence_is_signaled(entries[i].fence)) {
634 if (signaled_count == 0 && idx)
640 /* Initialize ret to the max of timeout and 1. That way, the
641 * default return value indicates a successful wait and not a
644 ret = max_t(signed long, timeout, 1);
646 if (signaled_count == count ||
647 (signaled_count > 0 &&
648 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
649 goto cleanup_entries;
651 /* There's a very annoying laxness in the dma_fence API here, in
652 * that backends are not required to automatically report when a
653 * fence is signaled prior to fence->ops->enable_signaling() being
654 * called. So here if we fail to match signaled_count, we need to
655 * fallthough and try a 0 timeout wait!
658 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
659 for (i = 0; i < count; ++i) {
660 if (entries[i].fence)
663 drm_syncobj_fence_get_or_add_callback(syncobjs[i],
665 &entries[i].syncobj_cb,
666 syncobj_wait_syncobj_func);
671 set_current_state(TASK_INTERRUPTIBLE);
674 for (i = 0; i < count; ++i) {
675 fence = entries[i].fence;
679 if (dma_fence_is_signaled(fence) ||
680 (!entries[i].fence_cb.func &&
681 dma_fence_add_callback(fence,
682 &entries[i].fence_cb,
683 syncobj_wait_fence_func))) {
684 /* The fence has been signaled */
685 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
695 if (signaled_count == count)
699 /* If we are doing a 0 timeout wait and we got
700 * here, then we just timed out.
706 ret = schedule_timeout(ret);
708 if (ret > 0 && signal_pending(current))
713 __set_current_state(TASK_RUNNING);
716 for (i = 0; i < count; ++i) {
717 if (entries[i].syncobj_cb.func)
718 drm_syncobj_remove_callback(syncobjs[i],
719 &entries[i].syncobj_cb);
720 if (entries[i].fence_cb.func)
721 dma_fence_remove_callback(entries[i].fence,
722 &entries[i].fence_cb);
723 dma_fence_put(entries[i].fence);
731 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
733 * @timeout_nsec: timeout nsec component in ns, 0 for poll
735 * Calculate the timeout in jiffies from an absolute time in sec/nsec.
737 static signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
739 ktime_t abs_timeout, now;
740 u64 timeout_ns, timeout_jiffies64;
742 /* make 0 timeout means poll - absolute 0 doesn't seem valid */
743 if (timeout_nsec == 0)
746 abs_timeout = ns_to_ktime(timeout_nsec);
749 if (!ktime_after(abs_timeout, now))
752 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
754 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
755 /* clamp timeout to avoid infinite timeout */
756 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
757 return MAX_SCHEDULE_TIMEOUT - 1;
759 return timeout_jiffies64 + 1;
762 static int drm_syncobj_array_wait(struct drm_device *dev,
763 struct drm_file *file_private,
764 struct drm_syncobj_wait *wait,
765 struct drm_syncobj **syncobjs)
767 signed long timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
771 ret = drm_syncobj_array_wait_timeout(syncobjs,
778 wait->first_signaled = first;
784 static int drm_syncobj_array_find(struct drm_file *file_private,
785 void *user_handles, uint32_t count_handles,
786 struct drm_syncobj ***syncobjs_out)
788 uint32_t i, *handles;
789 struct drm_syncobj **syncobjs;
792 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
796 if (copy_from_user(handles, user_handles,
797 sizeof(uint32_t) * count_handles)) {
799 goto err_free_handles;
802 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
803 if (syncobjs == NULL) {
805 goto err_free_handles;
808 for (i = 0; i < count_handles; i++) {
809 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
812 goto err_put_syncobjs;
817 *syncobjs_out = syncobjs;
822 drm_syncobj_put(syncobjs[i]);
830 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
834 for (i = 0; i < count; i++)
835 drm_syncobj_put(syncobjs[i]);
840 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
841 struct drm_file *file_private)
843 struct drm_syncobj_wait *args = data;
844 struct drm_syncobj **syncobjs;
847 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
850 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
851 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
854 if (args->count_handles == 0)
857 ret = drm_syncobj_array_find(file_private,
858 u64_to_user_ptr(args->handles),
864 ret = drm_syncobj_array_wait(dev, file_private,
867 drm_syncobj_array_free(syncobjs, args->count_handles);
873 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
874 struct drm_file *file_private)
876 struct drm_syncobj_array *args = data;
877 struct drm_syncobj **syncobjs;
881 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
887 if (args->count_handles == 0)
890 ret = drm_syncobj_array_find(file_private,
891 u64_to_user_ptr(args->handles),
897 for (i = 0; i < args->count_handles; i++)
898 drm_syncobj_replace_fence(syncobjs[i], NULL);
900 drm_syncobj_array_free(syncobjs, args->count_handles);
906 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
907 struct drm_file *file_private)
909 struct drm_syncobj_array *args = data;
910 struct drm_syncobj **syncobjs;
914 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
920 if (args->count_handles == 0)
923 ret = drm_syncobj_array_find(file_private,
924 u64_to_user_ptr(args->handles),
930 for (i = 0; i < args->count_handles; i++) {
931 ret = drm_syncobj_assign_null_handle(syncobjs[i]);
936 drm_syncobj_array_free(syncobjs, args->count_handles);