1 // SPDX-License-Identifier: MIT
4 * Copyright © 2019 Intel Corporation
7 #include <linux/delay.h>
8 #include <linux/dma-fence.h>
9 #include <linux/dma-fence-chain.h>
10 #include <linux/kernel.h>
11 #include <linux/kthread.h>
13 #include <linux/sched/signal.h>
14 #include <linux/slab.h>
15 #include <linux/spinlock.h>
16 #include <linux/random.h>
20 #define CHAIN_SZ (4 << 10)
22 static struct kmem_cache *slab_fences;
24 static inline struct mock_fence {
25 struct dma_fence base;
27 } *to_mock_fence(struct dma_fence *f) {
28 return container_of(f, struct mock_fence, base);
31 static const char *mock_name(struct dma_fence *f)
36 static void mock_fence_release(struct dma_fence *f)
38 kmem_cache_free(slab_fences, to_mock_fence(f));
41 static const struct dma_fence_ops mock_ops = {
42 .get_driver_name = mock_name,
43 .get_timeline_name = mock_name,
44 .release = mock_fence_release,
47 static struct dma_fence *mock_fence(void)
51 f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
55 spin_lock_init(&f->lock);
56 dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
61 static struct dma_fence *mock_chain(struct dma_fence *prev,
62 struct dma_fence *fence,
65 struct dma_fence_chain *f;
67 f = dma_fence_chain_alloc();
71 dma_fence_chain_init(f, dma_fence_get(prev), dma_fence_get(fence),
77 static int sanitycheck(void *arg)
79 struct dma_fence *f, *chain;
86 chain = mock_chain(NULL, f, 1);
88 dma_fence_enable_sw_signaling(chain);
100 struct fence_chains {
101 unsigned int chain_length;
102 struct dma_fence **fences;
103 struct dma_fence **chains;
105 struct dma_fence *tail;
108 static uint64_t seqno_inc(unsigned int i)
113 static int fence_chains_init(struct fence_chains *fc, unsigned int count,
114 uint64_t (*seqno_fn)(unsigned int))
119 fc->chains = kvmalloc_array(count, sizeof(*fc->chains),
120 GFP_KERNEL | __GFP_ZERO);
124 fc->fences = kvmalloc_array(count, sizeof(*fc->fences),
125 GFP_KERNEL | __GFP_ZERO);
132 for (i = 0; i < count; i++) {
133 fc->fences[i] = mock_fence();
134 if (!fc->fences[i]) {
139 fc->chains[i] = mock_chain(fc->tail,
142 if (!fc->chains[i]) {
147 fc->tail = fc->chains[i];
149 dma_fence_enable_sw_signaling(fc->chains[i]);
152 fc->chain_length = i;
156 for (i = 0; i < count; i++) {
157 dma_fence_put(fc->fences[i]);
158 dma_fence_put(fc->chains[i]);
166 static void fence_chains_fini(struct fence_chains *fc)
170 for (i = 0; i < fc->chain_length; i++) {
171 dma_fence_signal(fc->fences[i]);
172 dma_fence_put(fc->fences[i]);
176 for (i = 0; i < fc->chain_length; i++)
177 dma_fence_put(fc->chains[i]);
181 static int find_seqno(void *arg)
183 struct fence_chains fc;
184 struct dma_fence *fence;
188 err = fence_chains_init(&fc, 64, seqno_inc);
192 fence = dma_fence_get(fc.tail);
193 err = dma_fence_chain_find_seqno(&fence, 0);
194 dma_fence_put(fence);
196 pr_err("Reported %d for find_seqno(0)!\n", err);
200 for (i = 0; i < fc.chain_length; i++) {
201 fence = dma_fence_get(fc.tail);
202 err = dma_fence_chain_find_seqno(&fence, i + 1);
203 dma_fence_put(fence);
205 pr_err("Reported %d for find_seqno(%d:%d)!\n",
206 err, fc.chain_length + 1, i + 1);
209 if (fence != fc.chains[i]) {
210 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
211 fc.chain_length + 1, i + 1);
216 dma_fence_get(fence);
217 err = dma_fence_chain_find_seqno(&fence, i + 1);
218 dma_fence_put(fence);
220 pr_err("Error reported for finding self\n");
223 if (fence != fc.chains[i]) {
224 pr_err("Incorrect fence reported by find self\n");
229 dma_fence_get(fence);
230 err = dma_fence_chain_find_seqno(&fence, i + 2);
231 dma_fence_put(fence);
233 pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n",
239 dma_fence_get(fence);
240 err = dma_fence_chain_find_seqno(&fence, i);
241 dma_fence_put(fence);
243 pr_err("Error reported for previous fence!\n");
246 if (i > 0 && fence != fc.chains[i - 1]) {
247 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
255 fence_chains_fini(&fc);
259 static int find_signaled(void *arg)
261 struct fence_chains fc;
262 struct dma_fence *fence;
265 err = fence_chains_init(&fc, 2, seqno_inc);
269 dma_fence_signal(fc.fences[0]);
271 fence = dma_fence_get(fc.tail);
272 err = dma_fence_chain_find_seqno(&fence, 1);
273 dma_fence_put(fence);
275 pr_err("Reported %d for find_seqno()!\n", err);
279 if (fence && fence != fc.chains[0]) {
280 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n",
283 dma_fence_get(fence);
284 err = dma_fence_chain_find_seqno(&fence, 1);
285 dma_fence_put(fence);
287 pr_err("Reported %d for finding self!\n", err);
293 fence_chains_fini(&fc);
297 static int find_out_of_order(void *arg)
299 struct fence_chains fc;
300 struct dma_fence *fence;
303 err = fence_chains_init(&fc, 3, seqno_inc);
307 dma_fence_signal(fc.fences[1]);
309 fence = dma_fence_get(fc.tail);
310 err = dma_fence_chain_find_seqno(&fence, 2);
311 dma_fence_put(fence);
313 pr_err("Reported %d for find_seqno()!\n", err);
318 * We signaled the middle fence (2) of the 1-2-3 chain. The behavior
319 * of the dma-fence-chain is to make us wait for all the fences up to
320 * the point we want. Since fence 1 is still not signaled, this what
321 * we should get as fence to wait upon (fence 2 being garbage
322 * collected during the traversal of the chain).
324 if (fence != fc.chains[0]) {
325 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n",
326 fence ? fence->seqno : 0);
332 fence_chains_fini(&fc);
336 static uint64_t seqno_inc2(unsigned int i)
341 static int find_gap(void *arg)
343 struct fence_chains fc;
344 struct dma_fence *fence;
348 err = fence_chains_init(&fc, 64, seqno_inc2);
352 for (i = 0; i < fc.chain_length; i++) {
353 fence = dma_fence_get(fc.tail);
354 err = dma_fence_chain_find_seqno(&fence, 2 * i + 1);
355 dma_fence_put(fence);
357 pr_err("Reported %d for find_seqno(%d:%d)!\n",
358 err, fc.chain_length + 1, 2 * i + 1);
361 if (fence != fc.chains[i]) {
362 pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n",
370 dma_fence_get(fence);
371 err = dma_fence_chain_find_seqno(&fence, 2 * i + 2);
372 dma_fence_put(fence);
374 pr_err("Error reported for finding self\n");
377 if (fence != fc.chains[i]) {
378 pr_err("Incorrect fence reported by find self\n");
385 fence_chains_fini(&fc);
390 struct fence_chains fc;
394 static int __find_race(void *arg)
396 struct find_race *data = arg;
399 while (!kthread_should_stop()) {
400 struct dma_fence *fence = dma_fence_get(data->fc.tail);
403 seqno = get_random_u32_inclusive(1, data->fc.chain_length);
405 err = dma_fence_chain_find_seqno(&fence, seqno);
407 pr_err("Failed to find fence seqno:%d\n",
409 dma_fence_put(fence);
416 * We can only find ourselves if we are on fence we were
419 if (fence->seqno == seqno) {
420 err = dma_fence_chain_find_seqno(&fence, seqno);
422 pr_err("Reported an invalid fence for find-self:%d\n",
424 dma_fence_put(fence);
429 dma_fence_put(fence);
432 seqno = get_random_u32_below(data->fc.chain_length - 1);
433 dma_fence_signal(data->fc.fences[seqno]);
437 if (atomic_dec_and_test(&data->children))
438 wake_up_var(&data->children);
442 static int find_race(void *arg)
444 struct find_race data;
445 int ncpus = num_online_cpus();
446 struct task_struct **threads;
451 err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc);
455 threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL);
461 atomic_set(&data.children, 0);
462 for (i = 0; i < ncpus; i++) {
463 threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i);
464 if (IS_ERR(threads[i])) {
468 atomic_inc(&data.children);
469 get_task_struct(threads[i]);
472 wait_var_event_timeout(&data.children,
473 !atomic_read(&data.children),
476 for (i = 0; i < ncpus; i++) {
479 ret = kthread_stop_put(threads[i]);
486 for (i = 0; i < data.fc.chain_length; i++)
487 if (dma_fence_is_signaled(data.fc.fences[i]))
489 pr_info("Completed %lu cycles\n", count);
492 fence_chains_fini(&data.fc);
496 static int signal_forward(void *arg)
498 struct fence_chains fc;
502 err = fence_chains_init(&fc, 64, seqno_inc);
506 for (i = 0; i < fc.chain_length; i++) {
507 dma_fence_signal(fc.fences[i]);
509 if (!dma_fence_is_signaled(fc.chains[i])) {
510 pr_err("chain[%d] not signaled!\n", i);
515 if (i + 1 < fc.chain_length &&
516 dma_fence_is_signaled(fc.chains[i + 1])) {
517 pr_err("chain[%d] is signaled!\n", i);
524 fence_chains_fini(&fc);
528 static int signal_backward(void *arg)
530 struct fence_chains fc;
534 err = fence_chains_init(&fc, 64, seqno_inc);
538 for (i = fc.chain_length; i--; ) {
539 dma_fence_signal(fc.fences[i]);
541 if (i > 0 && dma_fence_is_signaled(fc.chains[i])) {
542 pr_err("chain[%d] is signaled!\n", i);
548 for (i = 0; i < fc.chain_length; i++) {
549 if (!dma_fence_is_signaled(fc.chains[i])) {
550 pr_err("chain[%d] was not signaled!\n", i);
557 fence_chains_fini(&fc);
561 static int __wait_fence_chains(void *arg)
563 struct fence_chains *fc = arg;
565 if (dma_fence_wait(fc->tail, false))
571 static int wait_forward(void *arg)
573 struct fence_chains fc;
574 struct task_struct *tsk;
578 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
582 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
587 get_task_struct(tsk);
590 for (i = 0; i < fc.chain_length; i++)
591 dma_fence_signal(fc.fences[i]);
593 err = kthread_stop_put(tsk);
596 fence_chains_fini(&fc);
600 static int wait_backward(void *arg)
602 struct fence_chains fc;
603 struct task_struct *tsk;
607 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
611 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
616 get_task_struct(tsk);
619 for (i = fc.chain_length; i--; )
620 dma_fence_signal(fc.fences[i]);
622 err = kthread_stop_put(tsk);
625 fence_chains_fini(&fc);
629 static void randomise_fences(struct fence_chains *fc)
631 unsigned int count = fc->chain_length;
633 /* Fisher-Yates shuffle courtesy of Knuth */
637 swp = get_random_u32_below(count + 1);
641 swap(fc->fences[count], fc->fences[swp]);
645 static int wait_random(void *arg)
647 struct fence_chains fc;
648 struct task_struct *tsk;
652 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
656 randomise_fences(&fc);
658 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
663 get_task_struct(tsk);
666 for (i = 0; i < fc.chain_length; i++)
667 dma_fence_signal(fc.fences[i]);
669 err = kthread_stop_put(tsk);
672 fence_chains_fini(&fc);
676 int dma_fence_chain(void)
678 static const struct subtest tests[] = {
679 SUBTEST(sanitycheck),
681 SUBTEST(find_signaled),
682 SUBTEST(find_out_of_order),
685 SUBTEST(signal_forward),
686 SUBTEST(signal_backward),
687 SUBTEST(wait_forward),
688 SUBTEST(wait_backward),
689 SUBTEST(wait_random),
693 pr_info("sizeof(dma_fence_chain)=%zu\n",
694 sizeof(struct dma_fence_chain));
696 slab_fences = KMEM_CACHE(mock_fence,
697 SLAB_TYPESAFE_BY_RCU |
702 ret = subtests(tests, NULL);
704 kmem_cache_destroy(slab_fences);