2 * DMA Engine test module
4 * Copyright (C) 2007 Atmel Corporation
5 * Copyright (C) 2013 Intel Corporation
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13 #include <linux/delay.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dmaengine.h>
16 #include <linux/freezer.h>
17 #include <linux/init.h>
18 #include <linux/kthread.h>
19 #include <linux/module.h>
20 #include <linux/moduleparam.h>
21 #include <linux/random.h>
22 #include <linux/slab.h>
23 #include <linux/wait.h>
25 static unsigned int test_buf_size = 16384;
26 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
27 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
29 static char test_channel[20];
30 module_param_string(channel, test_channel, sizeof(test_channel),
32 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
34 static char test_device[32];
35 module_param_string(device, test_device, sizeof(test_device),
37 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
39 static unsigned int threads_per_chan = 1;
40 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
41 MODULE_PARM_DESC(threads_per_chan,
42 "Number of threads to start per channel (default: 1)");
44 static unsigned int max_channels;
45 module_param(max_channels, uint, S_IRUGO | S_IWUSR);
46 MODULE_PARM_DESC(max_channels,
47 "Maximum number of channels to use (default: all)");
49 static unsigned int iterations;
50 module_param(iterations, uint, S_IRUGO | S_IWUSR);
51 MODULE_PARM_DESC(iterations,
52 "Iterations before stopping test (default: infinite)");
54 static unsigned int sg_buffers = 1;
55 module_param(sg_buffers, uint, S_IRUGO | S_IWUSR);
56 MODULE_PARM_DESC(sg_buffers,
57 "Number of scatter gather buffers (default: 1)");
59 static unsigned int dmatest;
60 module_param(dmatest, uint, S_IRUGO | S_IWUSR);
61 MODULE_PARM_DESC(dmatest,
62 "dmatest 0-memcpy 1-slave_sg (default: 0)");
64 static unsigned int xor_sources = 3;
65 module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
66 MODULE_PARM_DESC(xor_sources,
67 "Number of xor source buffers (default: 3)");
69 static unsigned int pq_sources = 3;
70 module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
71 MODULE_PARM_DESC(pq_sources,
72 "Number of p+q source buffers (default: 3)");
74 static int timeout = 3000;
75 module_param(timeout, uint, S_IRUGO | S_IWUSR);
76 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
77 "Pass -1 for infinite timeout");
80 module_param(noverify, bool, S_IRUGO | S_IWUSR);
81 MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
84 module_param(verbose, bool, S_IRUGO | S_IWUSR);
85 MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
88 * struct dmatest_params - test parameters.
89 * @buf_size: size of the memcpy test buffer
90 * @channel: bus ID of the channel to test
91 * @device: bus ID of the DMA Engine to test
92 * @threads_per_chan: number of threads to start per channel
93 * @max_channels: maximum number of channels to use
94 * @iterations: iterations before stopping test
95 * @xor_sources: number of xor source buffers
96 * @pq_sources: number of p+q source buffers
97 * @timeout: transfer timeout in msec, -1 for infinite timeout
99 struct dmatest_params {
100 unsigned int buf_size;
103 unsigned int threads_per_chan;
104 unsigned int max_channels;
105 unsigned int iterations;
106 unsigned int xor_sources;
107 unsigned int pq_sources;
113 * struct dmatest_info - test information.
114 * @params: test parameters
115 * @lock: access protection to the fields of this structure
117 static struct dmatest_info {
118 /* Test parameters */
119 struct dmatest_params params;
122 struct list_head channels;
123 unsigned int nr_channels;
127 .channels = LIST_HEAD_INIT(test_info.channels),
128 .lock = __MUTEX_INITIALIZER(test_info.lock),
131 static int dmatest_run_set(const char *val, const struct kernel_param *kp);
132 static int dmatest_run_get(char *val, const struct kernel_param *kp);
133 static const struct kernel_param_ops run_ops = {
134 .set = dmatest_run_set,
135 .get = dmatest_run_get,
137 static bool dmatest_run;
138 module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
139 MODULE_PARM_DESC(run, "Run the test (default: false)");
141 /* Maximum amount of mismatched bytes in buffer to print */
142 #define MAX_ERROR_COUNT 32
145 * Initialization patterns. All bytes in the source buffer has bit 7
146 * set, all bytes in the destination buffer has bit 7 cleared.
148 * Bit 6 is set for all bytes which are to be copied by the DMA
149 * engine. Bit 5 is set for all bytes which are to be overwritten by
152 * The remaining bits are the inverse of a counter which increments by
153 * one for each byte address.
155 #define PATTERN_SRC 0x80
156 #define PATTERN_DST 0x00
157 #define PATTERN_COPY 0x40
158 #define PATTERN_OVERWRITE 0x20
159 #define PATTERN_COUNT_MASK 0x1f
161 /* poor man's completion - we want to use wait_event_freezable() on it */
162 struct dmatest_done {
164 wait_queue_head_t *wait;
167 struct dmatest_thread {
168 struct list_head node;
169 struct dmatest_info *info;
170 struct task_struct *task;
171 struct dma_chan *chan;
174 enum dma_transaction_type type;
175 wait_queue_head_t done_wait;
176 struct dmatest_done test_done;
180 struct dmatest_chan {
181 struct list_head node;
182 struct dma_chan *chan;
183 struct list_head threads;
186 static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
189 static bool is_threaded_test_run(struct dmatest_info *info)
191 struct dmatest_chan *dtc;
193 list_for_each_entry(dtc, &info->channels, node) {
194 struct dmatest_thread *thread;
196 list_for_each_entry(thread, &dtc->threads, node) {
205 static int dmatest_wait_get(char *val, const struct kernel_param *kp)
207 struct dmatest_info *info = &test_info;
208 struct dmatest_params *params = &info->params;
210 if (params->iterations)
211 wait_event(thread_wait, !is_threaded_test_run(info));
213 return param_get_bool(val, kp);
216 static const struct kernel_param_ops wait_ops = {
217 .get = dmatest_wait_get,
218 .set = param_set_bool,
220 module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
221 MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
223 static bool dmatest_match_channel(struct dmatest_params *params,
224 struct dma_chan *chan)
226 if (params->channel[0] == '\0')
228 return strcmp(dma_chan_name(chan), params->channel) == 0;
231 static bool dmatest_match_device(struct dmatest_params *params,
232 struct dma_device *device)
234 if (params->device[0] == '\0')
236 return strcmp(dev_name(device->dev), params->device) == 0;
239 static unsigned long dmatest_random(void)
243 prandom_bytes(&buf, sizeof(buf));
247 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
248 unsigned int buf_size)
253 for (; (buf = *bufs); bufs++) {
254 for (i = 0; i < start; i++)
255 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
256 for ( ; i < start + len; i++)
257 buf[i] = PATTERN_SRC | PATTERN_COPY
258 | (~i & PATTERN_COUNT_MASK);
259 for ( ; i < buf_size; i++)
260 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
265 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
266 unsigned int buf_size)
271 for (; (buf = *bufs); bufs++) {
272 for (i = 0; i < start; i++)
273 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
274 for ( ; i < start + len; i++)
275 buf[i] = PATTERN_DST | PATTERN_OVERWRITE
276 | (~i & PATTERN_COUNT_MASK);
277 for ( ; i < buf_size; i++)
278 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
282 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
283 unsigned int counter, bool is_srcbuf)
285 u8 diff = actual ^ pattern;
286 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK);
287 const char *thread_name = current->comm;
290 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
291 thread_name, index, expected, actual);
292 else if ((pattern & PATTERN_COPY)
293 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
294 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
295 thread_name, index, expected, actual);
296 else if (diff & PATTERN_SRC)
297 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
298 thread_name, index, expected, actual);
300 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
301 thread_name, index, expected, actual);
304 static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
305 unsigned int end, unsigned int counter, u8 pattern,
309 unsigned int error_count = 0;
313 unsigned int counter_orig = counter;
315 for (; (buf = *bufs); bufs++) {
316 counter = counter_orig;
317 for (i = start; i < end; i++) {
319 expected = pattern | (~counter & PATTERN_COUNT_MASK);
320 if (actual != expected) {
321 if (error_count < MAX_ERROR_COUNT)
322 dmatest_mismatch(actual, pattern, i,
330 if (error_count > MAX_ERROR_COUNT)
331 pr_warn("%s: %u errors suppressed\n",
332 current->comm, error_count - MAX_ERROR_COUNT);
338 static void dmatest_callback(void *arg)
340 struct dmatest_done *done = arg;
341 struct dmatest_thread *thread =
342 container_of(done, struct dmatest_thread, test_done);
345 wake_up_all(done->wait);
348 * If thread->done, it means that this callback occurred
349 * after the parent thread has cleaned up. This can
350 * happen in the case that driver doesn't implement
351 * the terminate_all() functionality and a dma operation
352 * did not occur within the timeout period
354 WARN(1, "dmatest: Kernel memory may be corrupted!!\n");
358 static unsigned int min_odd(unsigned int x, unsigned int y)
360 unsigned int val = min(x, y);
362 return val % 2 ? val : val - 1;
365 static void result(const char *err, unsigned int n, unsigned int src_off,
366 unsigned int dst_off, unsigned int len, unsigned long data)
368 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
369 current->comm, n, err, src_off, dst_off, len, data);
372 static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
373 unsigned int dst_off, unsigned int len,
376 pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
377 current->comm, n, err, src_off, dst_off, len, data);
380 #define verbose_result(err, n, src_off, dst_off, len, data) ({ \
382 result(err, n, src_off, dst_off, len, data); \
384 dbg_result(err, n, src_off, dst_off, len, data);\
387 static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
389 unsigned long long per_sec = 1000000;
394 /* drop precision until runtime is 32-bits */
395 while (runtime > UINT_MAX) {
401 do_div(per_sec, runtime);
405 static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
407 return dmatest_persec(runtime, len >> 10);
411 * This function repeatedly tests DMA transfers of various lengths and
412 * offsets for a given operation type until it is told to exit by
413 * kthread_stop(). There may be multiple threads running this function
414 * in parallel for a single channel, and there may be multiple channels
415 * being tested in parallel.
417 * Before each test, the source and destination buffer is initialized
418 * with a known pattern. This pattern is different depending on
419 * whether it's in an area which is supposed to be copied or
420 * overwritten, and different in the source and destination buffers.
421 * So if the DMA engine doesn't copy exactly what we tell it to copy,
424 static int dmatest_func(void *data)
426 struct dmatest_thread *thread = data;
427 struct dmatest_done *done = &thread->test_done;
428 struct dmatest_info *info;
429 struct dmatest_params *params;
430 struct dma_chan *chan;
431 struct dma_device *dev;
432 unsigned int error_count;
433 unsigned int failed_tests = 0;
434 unsigned int total_tests = 0;
436 enum dma_status status;
437 enum dma_ctrl_flags flags;
443 ktime_t ktime, start, diff;
444 ktime_t filltime = ktime_set(0, 0);
445 ktime_t comparetime = ktime_set(0, 0);
447 unsigned long long total_len = 0;
455 params = &info->params;
458 if (thread->type == DMA_MEMCPY)
459 src_cnt = dst_cnt = 1;
460 else if (thread->type == DMA_SG)
461 src_cnt = dst_cnt = sg_buffers;
462 else if (thread->type == DMA_XOR) {
463 /* force odd to ensure dst = src */
464 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
466 } else if (thread->type == DMA_PQ) {
467 /* force odd to ensure dst = src */
468 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
471 pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
473 goto err_thread_type;
475 for (i = 0; i < src_cnt; i++)
478 goto err_thread_type;
480 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
483 for (i = 0; i < src_cnt; i++) {
484 thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
485 if (!thread->srcs[i])
488 thread->srcs[i] = NULL;
490 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
493 for (i = 0; i < dst_cnt; i++) {
494 thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
495 if (!thread->dsts[i])
498 thread->dsts[i] = NULL;
500 set_user_nice(current, 10);
503 * src and dst buffers are freed by ourselves below
505 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
508 while (!(kthread_should_stop() ||
509 (params->iterations && total_tests >= params->iterations))) {
510 struct dma_async_tx_descriptor *tx = NULL;
511 struct dmaengine_unmap_data *um;
512 dma_addr_t srcs[src_cnt];
514 unsigned int src_off, dst_off, len;
516 struct scatterlist tx_sg[src_cnt];
517 struct scatterlist rx_sg[src_cnt];
521 /* honor alignment restrictions */
522 if (thread->type == DMA_MEMCPY || thread->type == DMA_SG)
523 align = dev->copy_align;
524 else if (thread->type == DMA_XOR)
525 align = dev->xor_align;
526 else if (thread->type == DMA_PQ)
527 align = dev->pq_align;
529 if (1 << align > params->buf_size) {
530 pr_err("%u-byte buffer too small for %d-byte alignment\n",
531 params->buf_size, 1 << align);
535 if (params->noverify)
536 len = params->buf_size;
538 len = dmatest_random() % params->buf_size + 1;
540 len = (len >> align) << align;
546 if (params->noverify) {
551 src_off = dmatest_random() % (params->buf_size - len + 1);
552 dst_off = dmatest_random() % (params->buf_size - len + 1);
554 src_off = (src_off >> align) << align;
555 dst_off = (dst_off >> align) << align;
557 dmatest_init_srcs(thread->srcs, src_off, len,
559 dmatest_init_dsts(thread->dsts, dst_off, len,
562 diff = ktime_sub(ktime_get(), start);
563 filltime = ktime_add(filltime, diff);
566 um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt,
570 result("unmap data NULL", total_tests,
571 src_off, dst_off, len, ret);
575 um->len = params->buf_size;
576 for (i = 0; i < src_cnt; i++) {
577 void *buf = thread->srcs[i];
578 struct page *pg = virt_to_page(buf);
579 unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
581 um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
582 um->len, DMA_TO_DEVICE);
583 srcs[i] = um->addr[i] + src_off;
584 ret = dma_mapping_error(dev->dev, um->addr[i]);
586 result("src mapping error", total_tests,
587 src_off, dst_off, len, ret);
588 goto error_unmap_continue;
592 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
593 dsts = &um->addr[src_cnt];
594 for (i = 0; i < dst_cnt; i++) {
595 void *buf = thread->dsts[i];
596 struct page *pg = virt_to_page(buf);
597 unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
599 dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
601 ret = dma_mapping_error(dev->dev, dsts[i]);
603 result("dst mapping error", total_tests,
604 src_off, dst_off, len, ret);
605 goto error_unmap_continue;
610 sg_init_table(tx_sg, src_cnt);
611 sg_init_table(rx_sg, src_cnt);
612 for (i = 0; i < src_cnt; i++) {
613 sg_dma_address(&rx_sg[i]) = srcs[i];
614 sg_dma_address(&tx_sg[i]) = dsts[i] + dst_off;
615 sg_dma_len(&tx_sg[i]) = len;
616 sg_dma_len(&rx_sg[i]) = len;
619 if (thread->type == DMA_MEMCPY)
620 tx = dev->device_prep_dma_memcpy(chan,
622 srcs[0], len, flags);
623 else if (thread->type == DMA_SG)
624 tx = dev->device_prep_dma_sg(chan, tx_sg, src_cnt,
625 rx_sg, src_cnt, flags);
626 else if (thread->type == DMA_XOR)
627 tx = dev->device_prep_dma_xor(chan,
631 else if (thread->type == DMA_PQ) {
632 dma_addr_t dma_pq[dst_cnt];
634 for (i = 0; i < dst_cnt; i++)
635 dma_pq[i] = dsts[i] + dst_off;
636 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
642 result("prep error", total_tests, src_off,
645 goto error_unmap_continue;
649 tx->callback = dmatest_callback;
650 tx->callback_param = done;
651 cookie = tx->tx_submit(tx);
653 if (dma_submit_error(cookie)) {
654 result("submit error", total_tests, src_off,
657 goto error_unmap_continue;
659 dma_async_issue_pending(chan);
661 wait_event_freezable_timeout(thread->done_wait, done->done,
662 msecs_to_jiffies(params->timeout));
664 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
667 dmaengine_unmap_put(um);
668 result("test timed out", total_tests, src_off, dst_off,
670 goto error_unmap_continue;
671 } else if (status != DMA_COMPLETE) {
672 dmaengine_unmap_put(um);
673 result(status == DMA_ERROR ?
674 "completion error status" :
675 "completion busy status", total_tests, src_off,
677 goto error_unmap_continue;
680 dmaengine_unmap_put(um);
682 if (params->noverify) {
683 verbose_result("test passed", total_tests, src_off,
689 pr_debug("%s: verifying source buffer...\n", current->comm);
690 error_count = dmatest_verify(thread->srcs, 0, src_off,
691 0, PATTERN_SRC, true);
692 error_count += dmatest_verify(thread->srcs, src_off,
693 src_off + len, src_off,
694 PATTERN_SRC | PATTERN_COPY, true);
695 error_count += dmatest_verify(thread->srcs, src_off + len,
696 params->buf_size, src_off + len,
699 pr_debug("%s: verifying dest buffer...\n", current->comm);
700 error_count += dmatest_verify(thread->dsts, 0, dst_off,
701 0, PATTERN_DST, false);
702 error_count += dmatest_verify(thread->dsts, dst_off,
703 dst_off + len, src_off,
704 PATTERN_SRC | PATTERN_COPY, false);
705 error_count += dmatest_verify(thread->dsts, dst_off + len,
706 params->buf_size, dst_off + len,
709 diff = ktime_sub(ktime_get(), start);
710 comparetime = ktime_add(comparetime, diff);
713 result("data error", total_tests, src_off, dst_off,
717 verbose_result("test passed", total_tests, src_off,
723 error_unmap_continue:
724 dmaengine_unmap_put(um);
727 ktime = ktime_sub(ktime_get(), ktime);
728 ktime = ktime_sub(ktime, comparetime);
729 ktime = ktime_sub(ktime, filltime);
730 runtime = ktime_to_us(ktime);
734 for (i = 0; thread->dsts[i]; i++)
735 kfree(thread->dsts[i]);
739 for (i = 0; thread->srcs[i]; i++)
740 kfree(thread->srcs[i]);
745 pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
746 current->comm, total_tests, failed_tests,
747 dmatest_persec(runtime, total_tests),
748 dmatest_KBs(runtime, total_len), ret);
750 /* terminate all transfers on specified channels */
751 if (ret || failed_tests)
752 dmaengine_terminate_all(chan);
755 wake_up(&thread_wait);
760 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
762 struct dmatest_thread *thread;
763 struct dmatest_thread *_thread;
766 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
767 ret = kthread_stop(thread->task);
768 pr_debug("thread %s exited with status %d\n",
769 thread->task->comm, ret);
770 list_del(&thread->node);
771 put_task_struct(thread->task);
775 /* terminate all transfers on specified channels */
776 dmaengine_terminate_all(dtc->chan);
781 static int dmatest_add_threads(struct dmatest_info *info,
782 struct dmatest_chan *dtc, enum dma_transaction_type type)
784 struct dmatest_params *params = &info->params;
785 struct dmatest_thread *thread;
786 struct dma_chan *chan = dtc->chan;
790 if (type == DMA_MEMCPY)
792 else if (type == DMA_SG)
794 else if (type == DMA_XOR)
796 else if (type == DMA_PQ)
801 for (i = 0; i < params->threads_per_chan; i++) {
802 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
804 pr_warn("No memory for %s-%s%u\n",
805 dma_chan_name(chan), op, i);
809 thread->chan = dtc->chan;
811 thread->test_done.wait = &thread->done_wait;
812 init_waitqueue_head(&thread->done_wait);
814 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
815 dma_chan_name(chan), op, i);
816 if (IS_ERR(thread->task)) {
817 pr_warn("Failed to create thread %s-%s%u\n",
818 dma_chan_name(chan), op, i);
823 /* srcbuf and dstbuf are allocated by the thread itself */
824 get_task_struct(thread->task);
825 list_add_tail(&thread->node, &dtc->threads);
826 wake_up_process(thread->task);
832 static int dmatest_add_channel(struct dmatest_info *info,
833 struct dma_chan *chan)
835 struct dmatest_chan *dtc;
836 struct dma_device *dma_dev = chan->device;
837 unsigned int thread_count = 0;
840 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
842 pr_warn("No memory for %s\n", dma_chan_name(chan));
847 INIT_LIST_HEAD(&dtc->threads);
849 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
851 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
852 thread_count += cnt > 0 ? cnt : 0;
856 if (dma_has_cap(DMA_SG, dma_dev->cap_mask)) {
858 cnt = dmatest_add_threads(info, dtc, DMA_SG);
859 thread_count += cnt > 0 ? cnt : 0;
863 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
864 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
865 thread_count += cnt > 0 ? cnt : 0;
867 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
868 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
869 thread_count += cnt > 0 ? cnt : 0;
872 pr_info("Started %u threads using %s\n",
873 thread_count, dma_chan_name(chan));
875 list_add_tail(&dtc->node, &info->channels);
881 static bool filter(struct dma_chan *chan, void *param)
883 struct dmatest_params *params = param;
885 if (!dmatest_match_channel(params, chan) ||
886 !dmatest_match_device(params, chan->device))
892 static void request_channels(struct dmatest_info *info,
893 enum dma_transaction_type type)
898 dma_cap_set(type, mask);
900 struct dmatest_params *params = &info->params;
901 struct dma_chan *chan;
903 chan = dma_request_channel(mask, filter, params);
905 if (dmatest_add_channel(info, chan)) {
906 dma_release_channel(chan);
907 break; /* add_channel failed, punt */
910 break; /* no more channels available */
911 if (params->max_channels &&
912 info->nr_channels >= params->max_channels)
913 break; /* we have all we need */
917 static void run_threaded_test(struct dmatest_info *info)
919 struct dmatest_params *params = &info->params;
921 /* Copy test parameters */
922 params->buf_size = test_buf_size;
923 strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
924 strlcpy(params->device, strim(test_device), sizeof(params->device));
925 params->threads_per_chan = threads_per_chan;
926 params->max_channels = max_channels;
927 params->iterations = iterations;
928 params->xor_sources = xor_sources;
929 params->pq_sources = pq_sources;
930 params->timeout = timeout;
931 params->noverify = noverify;
933 request_channels(info, DMA_MEMCPY);
934 request_channels(info, DMA_XOR);
935 request_channels(info, DMA_SG);
936 request_channels(info, DMA_PQ);
939 static void stop_threaded_test(struct dmatest_info *info)
941 struct dmatest_chan *dtc, *_dtc;
942 struct dma_chan *chan;
944 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
945 list_del(&dtc->node);
947 dmatest_cleanup_channel(dtc);
948 pr_debug("dropped channel %s\n", dma_chan_name(chan));
949 dma_release_channel(chan);
952 info->nr_channels = 0;
955 static void restart_threaded_test(struct dmatest_info *info, bool run)
957 /* we might be called early to set run=, defer running until all
958 * parameters have been evaluated
963 /* Stop any running test first */
964 stop_threaded_test(info);
966 /* Run test with new parameters */
967 run_threaded_test(info);
970 static int dmatest_run_get(char *val, const struct kernel_param *kp)
972 struct dmatest_info *info = &test_info;
974 mutex_lock(&info->lock);
975 if (is_threaded_test_run(info)) {
978 stop_threaded_test(info);
981 mutex_unlock(&info->lock);
983 return param_get_bool(val, kp);
986 static int dmatest_run_set(const char *val, const struct kernel_param *kp)
988 struct dmatest_info *info = &test_info;
991 mutex_lock(&info->lock);
992 ret = param_set_bool(val, kp);
994 mutex_unlock(&info->lock);
998 if (is_threaded_test_run(info))
1000 else if (dmatest_run)
1001 restart_threaded_test(info, dmatest_run);
1003 mutex_unlock(&info->lock);
1008 static int __init dmatest_init(void)
1010 struct dmatest_info *info = &test_info;
1011 struct dmatest_params *params = &info->params;
1014 mutex_lock(&info->lock);
1015 run_threaded_test(info);
1016 mutex_unlock(&info->lock);
1019 if (params->iterations && wait)
1020 wait_event(thread_wait, !is_threaded_test_run(info));
1022 /* module parameters are stable, inittime tests are started,
1023 * let userspace take over 'run' control
1025 info->did_init = true;
1029 /* when compiled-in wait for drivers to load first */
1030 late_initcall(dmatest_init);
1032 static void __exit dmatest_exit(void)
1034 struct dmatest_info *info = &test_info;
1036 mutex_lock(&info->lock);
1037 stop_threaded_test(info);
1038 mutex_unlock(&info->lock);
1040 module_exit(dmatest_exit);
1042 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1043 MODULE_LICENSE("GPL v2");