GNU Linux-libre 5.19-rc6-gnu
[releases.git] / drivers / dma / dmatest.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * DMA Engine test module
4  *
5  * Copyright (C) 2007 Atmel Corporation
6  * Copyright (C) 2013 Intel Corporation
7  */
8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10 #include <linux/err.h>
11 #include <linux/delay.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmaengine.h>
14 #include <linux/freezer.h>
15 #include <linux/init.h>
16 #include <linux/kthread.h>
17 #include <linux/sched/task.h>
18 #include <linux/module.h>
19 #include <linux/moduleparam.h>
20 #include <linux/random.h>
21 #include <linux/slab.h>
22 #include <linux/wait.h>
23
24 static unsigned int test_buf_size = 16384;
25 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
26 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
27
28 static char test_device[32];
29 module_param_string(device, test_device, sizeof(test_device),
30                 S_IRUGO | S_IWUSR);
31 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
32
33 static unsigned int threads_per_chan = 1;
34 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
35 MODULE_PARM_DESC(threads_per_chan,
36                 "Number of threads to start per channel (default: 1)");
37
38 static unsigned int max_channels;
39 module_param(max_channels, uint, S_IRUGO | S_IWUSR);
40 MODULE_PARM_DESC(max_channels,
41                 "Maximum number of channels to use (default: all)");
42
43 static unsigned int iterations;
44 module_param(iterations, uint, S_IRUGO | S_IWUSR);
45 MODULE_PARM_DESC(iterations,
46                 "Iterations before stopping test (default: infinite)");
47
48 static unsigned int dmatest;
49 module_param(dmatest, uint, S_IRUGO | S_IWUSR);
50 MODULE_PARM_DESC(dmatest,
51                 "dmatest 0-memcpy 1-memset (default: 0)");
52
53 static unsigned int xor_sources = 3;
54 module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
55 MODULE_PARM_DESC(xor_sources,
56                 "Number of xor source buffers (default: 3)");
57
58 static unsigned int pq_sources = 3;
59 module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
60 MODULE_PARM_DESC(pq_sources,
61                 "Number of p+q source buffers (default: 3)");
62
63 static int timeout = 3000;
64 module_param(timeout, int, S_IRUGO | S_IWUSR);
65 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
66                  "Pass -1 for infinite timeout");
67
68 static bool noverify;
69 module_param(noverify, bool, S_IRUGO | S_IWUSR);
70 MODULE_PARM_DESC(noverify, "Disable data verification (default: verify)");
71
72 static bool norandom;
73 module_param(norandom, bool, 0644);
74 MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)");
75
76 static bool verbose;
77 module_param(verbose, bool, S_IRUGO | S_IWUSR);
78 MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
79
80 static int alignment = -1;
81 module_param(alignment, int, 0644);
82 MODULE_PARM_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
83
84 static unsigned int transfer_size;
85 module_param(transfer_size, uint, 0644);
86 MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))");
87
88 static bool polled;
89 module_param(polled, bool, S_IRUGO | S_IWUSR);
90 MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts");
91
92 /**
93  * struct dmatest_params - test parameters.
94  * @buf_size:           size of the memcpy test buffer
95  * @channel:            bus ID of the channel to test
96  * @device:             bus ID of the DMA Engine to test
97  * @threads_per_chan:   number of threads to start per channel
98  * @max_channels:       maximum number of channels to use
99  * @iterations:         iterations before stopping test
100  * @xor_sources:        number of xor source buffers
101  * @pq_sources:         number of p+q source buffers
102  * @timeout:            transfer timeout in msec, -1 for infinite timeout
103  * @noverify:           disable data verification
104  * @norandom:           disable random offset setup
105  * @alignment:          custom data address alignment taken as 2^alignment
106  * @transfer_size:      custom transfer size in bytes
107  * @polled:             use polling for completion instead of interrupts
108  */
109 struct dmatest_params {
110         unsigned int    buf_size;
111         char            channel[20];
112         char            device[32];
113         unsigned int    threads_per_chan;
114         unsigned int    max_channels;
115         unsigned int    iterations;
116         unsigned int    xor_sources;
117         unsigned int    pq_sources;
118         int             timeout;
119         bool            noverify;
120         bool            norandom;
121         int             alignment;
122         unsigned int    transfer_size;
123         bool            polled;
124 };
125
126 /**
127  * struct dmatest_info - test information.
128  * @params:             test parameters
129  * @channels:           channels under test
130  * @nr_channels:        number of channels under test
131  * @lock:               access protection to the fields of this structure
132  * @did_init:           module has been initialized completely
133  * @last_error:         test has faced configuration issues
134  */
135 static struct dmatest_info {
136         /* Test parameters */
137         struct dmatest_params   params;
138
139         /* Internal state */
140         struct list_head        channels;
141         unsigned int            nr_channels;
142         int                     last_error;
143         struct mutex            lock;
144         bool                    did_init;
145 } test_info = {
146         .channels = LIST_HEAD_INIT(test_info.channels),
147         .lock = __MUTEX_INITIALIZER(test_info.lock),
148 };
149
150 static int dmatest_run_set(const char *val, const struct kernel_param *kp);
151 static int dmatest_run_get(char *val, const struct kernel_param *kp);
152 static const struct kernel_param_ops run_ops = {
153         .set = dmatest_run_set,
154         .get = dmatest_run_get,
155 };
156 static bool dmatest_run;
157 module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
158 MODULE_PARM_DESC(run, "Run the test (default: false)");
159
160 static int dmatest_chan_set(const char *val, const struct kernel_param *kp);
161 static int dmatest_chan_get(char *val, const struct kernel_param *kp);
162 static const struct kernel_param_ops multi_chan_ops = {
163         .set = dmatest_chan_set,
164         .get = dmatest_chan_get,
165 };
166
167 static char test_channel[20];
168 static struct kparam_string newchan_kps = {
169         .string = test_channel,
170         .maxlen = 20,
171 };
172 module_param_cb(channel, &multi_chan_ops, &newchan_kps, 0644);
173 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
174
175 static int dmatest_test_list_get(char *val, const struct kernel_param *kp);
176 static const struct kernel_param_ops test_list_ops = {
177         .get = dmatest_test_list_get,
178 };
179 module_param_cb(test_list, &test_list_ops, NULL, 0444);
180 MODULE_PARM_DESC(test_list, "Print current test list");
181
182 /* Maximum amount of mismatched bytes in buffer to print */
183 #define MAX_ERROR_COUNT         32
184
185 /*
186  * Initialization patterns. All bytes in the source buffer has bit 7
187  * set, all bytes in the destination buffer has bit 7 cleared.
188  *
189  * Bit 6 is set for all bytes which are to be copied by the DMA
190  * engine. Bit 5 is set for all bytes which are to be overwritten by
191  * the DMA engine.
192  *
193  * The remaining bits are the inverse of a counter which increments by
194  * one for each byte address.
195  */
196 #define PATTERN_SRC             0x80
197 #define PATTERN_DST             0x00
198 #define PATTERN_COPY            0x40
199 #define PATTERN_OVERWRITE       0x20
200 #define PATTERN_COUNT_MASK      0x1f
201 #define PATTERN_MEMSET_IDX      0x01
202
203 /* Fixed point arithmetic ops */
204 #define FIXPT_SHIFT             8
205 #define FIXPNT_MASK             0xFF
206 #define FIXPT_TO_INT(a) ((a) >> FIXPT_SHIFT)
207 #define INT_TO_FIXPT(a) ((a) << FIXPT_SHIFT)
208 #define FIXPT_GET_FRAC(a)       ((((a) & FIXPNT_MASK) * 100) >> FIXPT_SHIFT)
209
210 /* poor man's completion - we want to use wait_event_freezable() on it */
211 struct dmatest_done {
212         bool                    done;
213         wait_queue_head_t       *wait;
214 };
215
216 struct dmatest_data {
217         u8              **raw;
218         u8              **aligned;
219         unsigned int    cnt;
220         unsigned int    off;
221 };
222
223 struct dmatest_thread {
224         struct list_head        node;
225         struct dmatest_info     *info;
226         struct task_struct      *task;
227         struct dma_chan         *chan;
228         struct dmatest_data     src;
229         struct dmatest_data     dst;
230         enum dma_transaction_type type;
231         wait_queue_head_t done_wait;
232         struct dmatest_done test_done;
233         bool                    done;
234         bool                    pending;
235 };
236
237 struct dmatest_chan {
238         struct list_head        node;
239         struct dma_chan         *chan;
240         struct list_head        threads;
241 };
242
243 static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
244 static bool wait;
245
246 static bool is_threaded_test_run(struct dmatest_info *info)
247 {
248         struct dmatest_chan *dtc;
249
250         list_for_each_entry(dtc, &info->channels, node) {
251                 struct dmatest_thread *thread;
252
253                 list_for_each_entry(thread, &dtc->threads, node) {
254                         if (!thread->done && !thread->pending)
255                                 return true;
256                 }
257         }
258
259         return false;
260 }
261
262 static bool is_threaded_test_pending(struct dmatest_info *info)
263 {
264         struct dmatest_chan *dtc;
265
266         list_for_each_entry(dtc, &info->channels, node) {
267                 struct dmatest_thread *thread;
268
269                 list_for_each_entry(thread, &dtc->threads, node) {
270                         if (thread->pending)
271                                 return true;
272                 }
273         }
274
275         return false;
276 }
277
278 static int dmatest_wait_get(char *val, const struct kernel_param *kp)
279 {
280         struct dmatest_info *info = &test_info;
281         struct dmatest_params *params = &info->params;
282
283         if (params->iterations)
284                 wait_event(thread_wait, !is_threaded_test_run(info));
285         wait = true;
286         return param_get_bool(val, kp);
287 }
288
289 static const struct kernel_param_ops wait_ops = {
290         .get = dmatest_wait_get,
291         .set = param_set_bool,
292 };
293 module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
294 MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
295
296 static bool dmatest_match_channel(struct dmatest_params *params,
297                 struct dma_chan *chan)
298 {
299         if (params->channel[0] == '\0')
300                 return true;
301         return strcmp(dma_chan_name(chan), params->channel) == 0;
302 }
303
304 static bool dmatest_match_device(struct dmatest_params *params,
305                 struct dma_device *device)
306 {
307         if (params->device[0] == '\0')
308                 return true;
309         return strcmp(dev_name(device->dev), params->device) == 0;
310 }
311
312 static unsigned long dmatest_random(void)
313 {
314         unsigned long buf;
315
316         prandom_bytes(&buf, sizeof(buf));
317         return buf;
318 }
319
320 static inline u8 gen_inv_idx(u8 index, bool is_memset)
321 {
322         u8 val = is_memset ? PATTERN_MEMSET_IDX : index;
323
324         return ~val & PATTERN_COUNT_MASK;
325 }
326
327 static inline u8 gen_src_value(u8 index, bool is_memset)
328 {
329         return PATTERN_SRC | gen_inv_idx(index, is_memset);
330 }
331
332 static inline u8 gen_dst_value(u8 index, bool is_memset)
333 {
334         return PATTERN_DST | gen_inv_idx(index, is_memset);
335 }
336
337 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
338                 unsigned int buf_size, bool is_memset)
339 {
340         unsigned int i;
341         u8 *buf;
342
343         for (; (buf = *bufs); bufs++) {
344                 for (i = 0; i < start; i++)
345                         buf[i] = gen_src_value(i, is_memset);
346                 for ( ; i < start + len; i++)
347                         buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;
348                 for ( ; i < buf_size; i++)
349                         buf[i] = gen_src_value(i, is_memset);
350                 buf++;
351         }
352 }
353
354 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
355                 unsigned int buf_size, bool is_memset)
356 {
357         unsigned int i;
358         u8 *buf;
359
360         for (; (buf = *bufs); bufs++) {
361                 for (i = 0; i < start; i++)
362                         buf[i] = gen_dst_value(i, is_memset);
363                 for ( ; i < start + len; i++)
364                         buf[i] = gen_dst_value(i, is_memset) |
365                                                 PATTERN_OVERWRITE;
366                 for ( ; i < buf_size; i++)
367                         buf[i] = gen_dst_value(i, is_memset);
368         }
369 }
370
371 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
372                 unsigned int counter, bool is_srcbuf, bool is_memset)
373 {
374         u8              diff = actual ^ pattern;
375         u8              expected = pattern | gen_inv_idx(counter, is_memset);
376         const char      *thread_name = current->comm;
377
378         if (is_srcbuf)
379                 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
380                         thread_name, index, expected, actual);
381         else if ((pattern & PATTERN_COPY)
382                         && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
383                 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
384                         thread_name, index, expected, actual);
385         else if (diff & PATTERN_SRC)
386                 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
387                         thread_name, index, expected, actual);
388         else
389                 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
390                         thread_name, index, expected, actual);
391 }
392
393 static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
394                 unsigned int end, unsigned int counter, u8 pattern,
395                 bool is_srcbuf, bool is_memset)
396 {
397         unsigned int i;
398         unsigned int error_count = 0;
399         u8 actual;
400         u8 expected;
401         u8 *buf;
402         unsigned int counter_orig = counter;
403
404         for (; (buf = *bufs); bufs++) {
405                 counter = counter_orig;
406                 for (i = start; i < end; i++) {
407                         actual = buf[i];
408                         expected = pattern | gen_inv_idx(counter, is_memset);
409                         if (actual != expected) {
410                                 if (error_count < MAX_ERROR_COUNT)
411                                         dmatest_mismatch(actual, pattern, i,
412                                                          counter, is_srcbuf,
413                                                          is_memset);
414                                 error_count++;
415                         }
416                         counter++;
417                 }
418         }
419
420         if (error_count > MAX_ERROR_COUNT)
421                 pr_warn("%s: %u errors suppressed\n",
422                         current->comm, error_count - MAX_ERROR_COUNT);
423
424         return error_count;
425 }
426
427
428 static void dmatest_callback(void *arg)
429 {
430         struct dmatest_done *done = arg;
431         struct dmatest_thread *thread =
432                 container_of(done, struct dmatest_thread, test_done);
433         if (!thread->done) {
434                 done->done = true;
435                 wake_up_all(done->wait);
436         } else {
437                 /*
438                  * If thread->done, it means that this callback occurred
439                  * after the parent thread has cleaned up. This can
440                  * happen in the case that driver doesn't implement
441                  * the terminate_all() functionality and a dma operation
442                  * did not occur within the timeout period
443                  */
444                 WARN(1, "dmatest: Kernel memory may be corrupted!!\n");
445         }
446 }
447
448 static unsigned int min_odd(unsigned int x, unsigned int y)
449 {
450         unsigned int val = min(x, y);
451
452         return val % 2 ? val : val - 1;
453 }
454
455 static void result(const char *err, unsigned int n, unsigned int src_off,
456                    unsigned int dst_off, unsigned int len, unsigned long data)
457 {
458         if (IS_ERR_VALUE(data)) {
459                 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%ld)\n",
460                         current->comm, n, err, src_off, dst_off, len, data);
461         } else {
462                 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
463                         current->comm, n, err, src_off, dst_off, len, data);
464         }
465 }
466
467 static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
468                        unsigned int dst_off, unsigned int len,
469                        unsigned long data)
470 {
471         pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
472                  current->comm, n, err, src_off, dst_off, len, data);
473 }
474
475 #define verbose_result(err, n, src_off, dst_off, len, data) ({  \
476         if (verbose)                                            \
477                 result(err, n, src_off, dst_off, len, data);    \
478         else                                                    \
479                 dbg_result(err, n, src_off, dst_off, len, data);\
480 })
481
482 static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
483 {
484         unsigned long long per_sec = 1000000;
485
486         if (runtime <= 0)
487                 return 0;
488
489         /* drop precision until runtime is 32-bits */
490         while (runtime > UINT_MAX) {
491                 runtime >>= 1;
492                 per_sec <<= 1;
493         }
494
495         per_sec *= val;
496         per_sec = INT_TO_FIXPT(per_sec);
497         do_div(per_sec, runtime);
498
499         return per_sec;
500 }
501
502 static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
503 {
504         return FIXPT_TO_INT(dmatest_persec(runtime, len >> 10));
505 }
506
507 static void __dmatest_free_test_data(struct dmatest_data *d, unsigned int cnt)
508 {
509         unsigned int i;
510
511         for (i = 0; i < cnt; i++)
512                 kfree(d->raw[i]);
513
514         kfree(d->aligned);
515         kfree(d->raw);
516 }
517
518 static void dmatest_free_test_data(struct dmatest_data *d)
519 {
520         __dmatest_free_test_data(d, d->cnt);
521 }
522
523 static int dmatest_alloc_test_data(struct dmatest_data *d,
524                 unsigned int buf_size, u8 align)
525 {
526         unsigned int i = 0;
527
528         d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
529         if (!d->raw)
530                 return -ENOMEM;
531
532         d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
533         if (!d->aligned)
534                 goto err;
535
536         for (i = 0; i < d->cnt; i++) {
537                 d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL);
538                 if (!d->raw[i])
539                         goto err;
540
541                 /* align to alignment restriction */
542                 if (align)
543                         d->aligned[i] = PTR_ALIGN(d->raw[i], align);
544                 else
545                         d->aligned[i] = d->raw[i];
546         }
547
548         return 0;
549 err:
550         __dmatest_free_test_data(d, i);
551         return -ENOMEM;
552 }
553
554 /*
555  * This function repeatedly tests DMA transfers of various lengths and
556  * offsets for a given operation type until it is told to exit by
557  * kthread_stop(). There may be multiple threads running this function
558  * in parallel for a single channel, and there may be multiple channels
559  * being tested in parallel.
560  *
561  * Before each test, the source and destination buffer is initialized
562  * with a known pattern. This pattern is different depending on
563  * whether it's in an area which is supposed to be copied or
564  * overwritten, and different in the source and destination buffers.
565  * So if the DMA engine doesn't copy exactly what we tell it to copy,
566  * we'll notice.
567  */
568 static int dmatest_func(void *data)
569 {
570         struct dmatest_thread   *thread = data;
571         struct dmatest_done     *done = &thread->test_done;
572         struct dmatest_info     *info;
573         struct dmatest_params   *params;
574         struct dma_chan         *chan;
575         struct dma_device       *dev;
576         struct device           *dma_dev;
577         unsigned int            error_count;
578         unsigned int            failed_tests = 0;
579         unsigned int            total_tests = 0;
580         dma_cookie_t            cookie;
581         enum dma_status         status;
582         enum dma_ctrl_flags     flags;
583         u8                      *pq_coefs = NULL;
584         int                     ret;
585         unsigned int            buf_size;
586         struct dmatest_data     *src;
587         struct dmatest_data     *dst;
588         int                     i;
589         ktime_t                 ktime, start, diff;
590         ktime_t                 filltime = 0;
591         ktime_t                 comparetime = 0;
592         s64                     runtime = 0;
593         unsigned long long      total_len = 0;
594         unsigned long long      iops = 0;
595         u8                      align = 0;
596         bool                    is_memset = false;
597         dma_addr_t              *srcs;
598         dma_addr_t              *dma_pq;
599
600         set_freezable();
601
602         ret = -ENOMEM;
603
604         smp_rmb();
605         thread->pending = false;
606         info = thread->info;
607         params = &info->params;
608         chan = thread->chan;
609         dev = chan->device;
610         dma_dev = dmaengine_get_dma_device(chan);
611
612         src = &thread->src;
613         dst = &thread->dst;
614         if (thread->type == DMA_MEMCPY) {
615                 align = params->alignment < 0 ? dev->copy_align :
616                                                 params->alignment;
617                 src->cnt = dst->cnt = 1;
618         } else if (thread->type == DMA_MEMSET) {
619                 align = params->alignment < 0 ? dev->fill_align :
620                                                 params->alignment;
621                 src->cnt = dst->cnt = 1;
622                 is_memset = true;
623         } else if (thread->type == DMA_XOR) {
624                 /* force odd to ensure dst = src */
625                 src->cnt = min_odd(params->xor_sources | 1, dev->max_xor);
626                 dst->cnt = 1;
627                 align = params->alignment < 0 ? dev->xor_align :
628                                                 params->alignment;
629         } else if (thread->type == DMA_PQ) {
630                 /* force odd to ensure dst = src */
631                 src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
632                 dst->cnt = 2;
633                 align = params->alignment < 0 ? dev->pq_align :
634                                                 params->alignment;
635
636                 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
637                 if (!pq_coefs)
638                         goto err_thread_type;
639
640                 for (i = 0; i < src->cnt; i++)
641                         pq_coefs[i] = 1;
642         } else
643                 goto err_thread_type;
644
645         /* Check if buffer count fits into map count variable (u8) */
646         if ((src->cnt + dst->cnt) >= 255) {
647                 pr_err("too many buffers (%d of 255 supported)\n",
648                        src->cnt + dst->cnt);
649                 goto err_free_coefs;
650         }
651
652         buf_size = params->buf_size;
653         if (1 << align > buf_size) {
654                 pr_err("%u-byte buffer too small for %d-byte alignment\n",
655                        buf_size, 1 << align);
656                 goto err_free_coefs;
657         }
658
659         if (dmatest_alloc_test_data(src, buf_size, align) < 0)
660                 goto err_free_coefs;
661
662         if (dmatest_alloc_test_data(dst, buf_size, align) < 0)
663                 goto err_src;
664
665         set_user_nice(current, 10);
666
667         srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL);
668         if (!srcs)
669                 goto err_dst;
670
671         dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL);
672         if (!dma_pq)
673                 goto err_srcs_array;
674
675         /*
676          * src and dst buffers are freed by ourselves below
677          */
678         if (params->polled)
679                 flags = DMA_CTRL_ACK;
680         else
681                 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
682
683         ktime = ktime_get();
684         while (!(kthread_should_stop() ||
685                (params->iterations && total_tests >= params->iterations))) {
686                 struct dma_async_tx_descriptor *tx = NULL;
687                 struct dmaengine_unmap_data *um;
688                 dma_addr_t *dsts;
689                 unsigned int len;
690
691                 total_tests++;
692
693                 if (params->transfer_size) {
694                         if (params->transfer_size >= buf_size) {
695                                 pr_err("%u-byte transfer size must be lower than %u-buffer size\n",
696                                        params->transfer_size, buf_size);
697                                 break;
698                         }
699                         len = params->transfer_size;
700                 } else if (params->norandom) {
701                         len = buf_size;
702                 } else {
703                         len = dmatest_random() % buf_size + 1;
704                 }
705
706                 /* Do not alter transfer size explicitly defined by user */
707                 if (!params->transfer_size) {
708                         len = (len >> align) << align;
709                         if (!len)
710                                 len = 1 << align;
711                 }
712                 total_len += len;
713
714                 if (params->norandom) {
715                         src->off = 0;
716                         dst->off = 0;
717                 } else {
718                         src->off = dmatest_random() % (buf_size - len + 1);
719                         dst->off = dmatest_random() % (buf_size - len + 1);
720
721                         src->off = (src->off >> align) << align;
722                         dst->off = (dst->off >> align) << align;
723                 }
724
725                 if (!params->noverify) {
726                         start = ktime_get();
727                         dmatest_init_srcs(src->aligned, src->off, len,
728                                           buf_size, is_memset);
729                         dmatest_init_dsts(dst->aligned, dst->off, len,
730                                           buf_size, is_memset);
731
732                         diff = ktime_sub(ktime_get(), start);
733                         filltime = ktime_add(filltime, diff);
734                 }
735
736                 um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt,
737                                               GFP_KERNEL);
738                 if (!um) {
739                         failed_tests++;
740                         result("unmap data NULL", total_tests,
741                                src->off, dst->off, len, ret);
742                         continue;
743                 }
744
745                 um->len = buf_size;
746                 for (i = 0; i < src->cnt; i++) {
747                         void *buf = src->aligned[i];
748                         struct page *pg = virt_to_page(buf);
749                         unsigned long pg_off = offset_in_page(buf);
750
751                         um->addr[i] = dma_map_page(dma_dev, pg, pg_off,
752                                                    um->len, DMA_TO_DEVICE);
753                         srcs[i] = um->addr[i] + src->off;
754                         ret = dma_mapping_error(dma_dev, um->addr[i]);
755                         if (ret) {
756                                 result("src mapping error", total_tests,
757                                        src->off, dst->off, len, ret);
758                                 goto error_unmap_continue;
759                         }
760                         um->to_cnt++;
761                 }
762                 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
763                 dsts = &um->addr[src->cnt];
764                 for (i = 0; i < dst->cnt; i++) {
765                         void *buf = dst->aligned[i];
766                         struct page *pg = virt_to_page(buf);
767                         unsigned long pg_off = offset_in_page(buf);
768
769                         dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len,
770                                                DMA_BIDIRECTIONAL);
771                         ret = dma_mapping_error(dma_dev, dsts[i]);
772                         if (ret) {
773                                 result("dst mapping error", total_tests,
774                                        src->off, dst->off, len, ret);
775                                 goto error_unmap_continue;
776                         }
777                         um->bidi_cnt++;
778                 }
779
780                 if (thread->type == DMA_MEMCPY)
781                         tx = dev->device_prep_dma_memcpy(chan,
782                                                          dsts[0] + dst->off,
783                                                          srcs[0], len, flags);
784                 else if (thread->type == DMA_MEMSET)
785                         tx = dev->device_prep_dma_memset(chan,
786                                                 dsts[0] + dst->off,
787                                                 *(src->aligned[0] + src->off),
788                                                 len, flags);
789                 else if (thread->type == DMA_XOR)
790                         tx = dev->device_prep_dma_xor(chan,
791                                                       dsts[0] + dst->off,
792                                                       srcs, src->cnt,
793                                                       len, flags);
794                 else if (thread->type == DMA_PQ) {
795                         for (i = 0; i < dst->cnt; i++)
796                                 dma_pq[i] = dsts[i] + dst->off;
797                         tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
798                                                      src->cnt, pq_coefs,
799                                                      len, flags);
800                 }
801
802                 if (!tx) {
803                         result("prep error", total_tests, src->off,
804                                dst->off, len, ret);
805                         msleep(100);
806                         goto error_unmap_continue;
807                 }
808
809                 done->done = false;
810                 if (!params->polled) {
811                         tx->callback = dmatest_callback;
812                         tx->callback_param = done;
813                 }
814                 cookie = tx->tx_submit(tx);
815
816                 if (dma_submit_error(cookie)) {
817                         result("submit error", total_tests, src->off,
818                                dst->off, len, ret);
819                         msleep(100);
820                         goto error_unmap_continue;
821                 }
822
823                 if (params->polled) {
824                         status = dma_sync_wait(chan, cookie);
825                         dmaengine_terminate_sync(chan);
826                         if (status == DMA_COMPLETE)
827                                 done->done = true;
828                 } else {
829                         dma_async_issue_pending(chan);
830
831                         wait_event_freezable_timeout(thread->done_wait,
832                                         done->done,
833                                         msecs_to_jiffies(params->timeout));
834
835                         status = dma_async_is_tx_complete(chan, cookie, NULL,
836                                                           NULL);
837                 }
838
839                 if (!done->done) {
840                         result("test timed out", total_tests, src->off, dst->off,
841                                len, 0);
842                         goto error_unmap_continue;
843                 } else if (status != DMA_COMPLETE &&
844                            !(dma_has_cap(DMA_COMPLETION_NO_ORDER,
845                                          dev->cap_mask) &&
846                              status == DMA_OUT_OF_ORDER)) {
847                         result(status == DMA_ERROR ?
848                                "completion error status" :
849                                "completion busy status", total_tests, src->off,
850                                dst->off, len, ret);
851                         goto error_unmap_continue;
852                 }
853
854                 dmaengine_unmap_put(um);
855
856                 if (params->noverify) {
857                         verbose_result("test passed", total_tests, src->off,
858                                        dst->off, len, 0);
859                         continue;
860                 }
861
862                 start = ktime_get();
863                 pr_debug("%s: verifying source buffer...\n", current->comm);
864                 error_count = dmatest_verify(src->aligned, 0, src->off,
865                                 0, PATTERN_SRC, true, is_memset);
866                 error_count += dmatest_verify(src->aligned, src->off,
867                                 src->off + len, src->off,
868                                 PATTERN_SRC | PATTERN_COPY, true, is_memset);
869                 error_count += dmatest_verify(src->aligned, src->off + len,
870                                 buf_size, src->off + len,
871                                 PATTERN_SRC, true, is_memset);
872
873                 pr_debug("%s: verifying dest buffer...\n", current->comm);
874                 error_count += dmatest_verify(dst->aligned, 0, dst->off,
875                                 0, PATTERN_DST, false, is_memset);
876
877                 error_count += dmatest_verify(dst->aligned, dst->off,
878                                 dst->off + len, src->off,
879                                 PATTERN_SRC | PATTERN_COPY, false, is_memset);
880
881                 error_count += dmatest_verify(dst->aligned, dst->off + len,
882                                 buf_size, dst->off + len,
883                                 PATTERN_DST, false, is_memset);
884
885                 diff = ktime_sub(ktime_get(), start);
886                 comparetime = ktime_add(comparetime, diff);
887
888                 if (error_count) {
889                         result("data error", total_tests, src->off, dst->off,
890                                len, error_count);
891                         failed_tests++;
892                 } else {
893                         verbose_result("test passed", total_tests, src->off,
894                                        dst->off, len, 0);
895                 }
896
897                 continue;
898
899 error_unmap_continue:
900                 dmaengine_unmap_put(um);
901                 failed_tests++;
902         }
903         ktime = ktime_sub(ktime_get(), ktime);
904         ktime = ktime_sub(ktime, comparetime);
905         ktime = ktime_sub(ktime, filltime);
906         runtime = ktime_to_us(ktime);
907
908         ret = 0;
909         kfree(dma_pq);
910 err_srcs_array:
911         kfree(srcs);
912 err_dst:
913         dmatest_free_test_data(dst);
914 err_src:
915         dmatest_free_test_data(src);
916 err_free_coefs:
917         kfree(pq_coefs);
918 err_thread_type:
919         iops = dmatest_persec(runtime, total_tests);
920         pr_info("%s: summary %u tests, %u failures %llu.%02llu iops %llu KB/s (%d)\n",
921                 current->comm, total_tests, failed_tests,
922                 FIXPT_TO_INT(iops), FIXPT_GET_FRAC(iops),
923                 dmatest_KBs(runtime, total_len), ret);
924
925         /* terminate all transfers on specified channels */
926         if (ret || failed_tests)
927                 dmaengine_terminate_sync(chan);
928
929         thread->done = true;
930         wake_up(&thread_wait);
931
932         return ret;
933 }
934
935 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
936 {
937         struct dmatest_thread   *thread;
938         struct dmatest_thread   *_thread;
939         int                     ret;
940
941         list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
942                 ret = kthread_stop(thread->task);
943                 pr_debug("thread %s exited with status %d\n",
944                          thread->task->comm, ret);
945                 list_del(&thread->node);
946                 put_task_struct(thread->task);
947                 kfree(thread);
948         }
949
950         /* terminate all transfers on specified channels */
951         dmaengine_terminate_sync(dtc->chan);
952
953         kfree(dtc);
954 }
955
956 static int dmatest_add_threads(struct dmatest_info *info,
957                 struct dmatest_chan *dtc, enum dma_transaction_type type)
958 {
959         struct dmatest_params *params = &info->params;
960         struct dmatest_thread *thread;
961         struct dma_chan *chan = dtc->chan;
962         char *op;
963         unsigned int i;
964
965         if (type == DMA_MEMCPY)
966                 op = "copy";
967         else if (type == DMA_MEMSET)
968                 op = "set";
969         else if (type == DMA_XOR)
970                 op = "xor";
971         else if (type == DMA_PQ)
972                 op = "pq";
973         else
974                 return -EINVAL;
975
976         for (i = 0; i < params->threads_per_chan; i++) {
977                 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
978                 if (!thread) {
979                         pr_warn("No memory for %s-%s%u\n",
980                                 dma_chan_name(chan), op, i);
981                         break;
982                 }
983                 thread->info = info;
984                 thread->chan = dtc->chan;
985                 thread->type = type;
986                 thread->test_done.wait = &thread->done_wait;
987                 init_waitqueue_head(&thread->done_wait);
988                 smp_wmb();
989                 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
990                                 dma_chan_name(chan), op, i);
991                 if (IS_ERR(thread->task)) {
992                         pr_warn("Failed to create thread %s-%s%u\n",
993                                 dma_chan_name(chan), op, i);
994                         kfree(thread);
995                         break;
996                 }
997
998                 /* srcbuf and dstbuf are allocated by the thread itself */
999                 get_task_struct(thread->task);
1000                 list_add_tail(&thread->node, &dtc->threads);
1001                 thread->pending = true;
1002         }
1003
1004         return i;
1005 }
1006
1007 static int dmatest_add_channel(struct dmatest_info *info,
1008                 struct dma_chan *chan)
1009 {
1010         struct dmatest_chan     *dtc;
1011         struct dma_device       *dma_dev = chan->device;
1012         unsigned int            thread_count = 0;
1013         int cnt;
1014
1015         dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
1016         if (!dtc) {
1017                 pr_warn("No memory for %s\n", dma_chan_name(chan));
1018                 return -ENOMEM;
1019         }
1020
1021         dtc->chan = chan;
1022         INIT_LIST_HEAD(&dtc->threads);
1023
1024         if (dma_has_cap(DMA_COMPLETION_NO_ORDER, dma_dev->cap_mask) &&
1025             info->params.polled) {
1026                 info->params.polled = false;
1027                 pr_warn("DMA_COMPLETION_NO_ORDER, polled disabled\n");
1028         }
1029
1030         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1031                 if (dmatest == 0) {
1032                         cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
1033                         thread_count += cnt > 0 ? cnt : 0;
1034                 }
1035         }
1036
1037         if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1038                 if (dmatest == 1) {
1039                         cnt = dmatest_add_threads(info, dtc, DMA_MEMSET);
1040                         thread_count += cnt > 0 ? cnt : 0;
1041                 }
1042         }
1043
1044         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1045                 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
1046                 thread_count += cnt > 0 ? cnt : 0;
1047         }
1048         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1049                 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
1050                 thread_count += cnt > 0 ? cnt : 0;
1051         }
1052
1053         pr_info("Added %u threads using %s\n",
1054                 thread_count, dma_chan_name(chan));
1055
1056         list_add_tail(&dtc->node, &info->channels);
1057         info->nr_channels++;
1058
1059         return 0;
1060 }
1061
1062 static bool filter(struct dma_chan *chan, void *param)
1063 {
1064         return dmatest_match_channel(param, chan) && dmatest_match_device(param, chan->device);
1065 }
1066
1067 static void request_channels(struct dmatest_info *info,
1068                              enum dma_transaction_type type)
1069 {
1070         dma_cap_mask_t mask;
1071
1072         dma_cap_zero(mask);
1073         dma_cap_set(type, mask);
1074         for (;;) {
1075                 struct dmatest_params *params = &info->params;
1076                 struct dma_chan *chan;
1077
1078                 chan = dma_request_channel(mask, filter, params);
1079                 if (chan) {
1080                         if (dmatest_add_channel(info, chan)) {
1081                                 dma_release_channel(chan);
1082                                 break; /* add_channel failed, punt */
1083                         }
1084                 } else
1085                         break; /* no more channels available */
1086                 if (params->max_channels &&
1087                     info->nr_channels >= params->max_channels)
1088                         break; /* we have all we need */
1089         }
1090 }
1091
1092 static void add_threaded_test(struct dmatest_info *info)
1093 {
1094         struct dmatest_params *params = &info->params;
1095
1096         /* Copy test parameters */
1097         params->buf_size = test_buf_size;
1098         strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
1099         strlcpy(params->device, strim(test_device), sizeof(params->device));
1100         params->threads_per_chan = threads_per_chan;
1101         params->max_channels = max_channels;
1102         params->iterations = iterations;
1103         params->xor_sources = xor_sources;
1104         params->pq_sources = pq_sources;
1105         params->timeout = timeout;
1106         params->noverify = noverify;
1107         params->norandom = norandom;
1108         params->alignment = alignment;
1109         params->transfer_size = transfer_size;
1110         params->polled = polled;
1111
1112         request_channels(info, DMA_MEMCPY);
1113         request_channels(info, DMA_MEMSET);
1114         request_channels(info, DMA_XOR);
1115         request_channels(info, DMA_PQ);
1116 }
1117
1118 static void run_pending_tests(struct dmatest_info *info)
1119 {
1120         struct dmatest_chan *dtc;
1121         unsigned int thread_count = 0;
1122
1123         list_for_each_entry(dtc, &info->channels, node) {
1124                 struct dmatest_thread *thread;
1125
1126                 thread_count = 0;
1127                 list_for_each_entry(thread, &dtc->threads, node) {
1128                         wake_up_process(thread->task);
1129                         thread_count++;
1130                 }
1131                 pr_info("Started %u threads using %s\n",
1132                         thread_count, dma_chan_name(dtc->chan));
1133         }
1134 }
1135
1136 static void stop_threaded_test(struct dmatest_info *info)
1137 {
1138         struct dmatest_chan *dtc, *_dtc;
1139         struct dma_chan *chan;
1140
1141         list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
1142                 list_del(&dtc->node);
1143                 chan = dtc->chan;
1144                 dmatest_cleanup_channel(dtc);
1145                 pr_debug("dropped channel %s\n", dma_chan_name(chan));
1146                 dma_release_channel(chan);
1147         }
1148
1149         info->nr_channels = 0;
1150 }
1151
1152 static void start_threaded_tests(struct dmatest_info *info)
1153 {
1154         /* we might be called early to set run=, defer running until all
1155          * parameters have been evaluated
1156          */
1157         if (!info->did_init)
1158                 return;
1159
1160         run_pending_tests(info);
1161 }
1162
1163 static int dmatest_run_get(char *val, const struct kernel_param *kp)
1164 {
1165         struct dmatest_info *info = &test_info;
1166
1167         mutex_lock(&info->lock);
1168         if (is_threaded_test_run(info)) {
1169                 dmatest_run = true;
1170         } else {
1171                 if (!is_threaded_test_pending(info))
1172                         stop_threaded_test(info);
1173                 dmatest_run = false;
1174         }
1175         mutex_unlock(&info->lock);
1176
1177         return param_get_bool(val, kp);
1178 }
1179
1180 static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1181 {
1182         struct dmatest_info *info = &test_info;
1183         int ret;
1184
1185         mutex_lock(&info->lock);
1186         ret = param_set_bool(val, kp);
1187         if (ret) {
1188                 mutex_unlock(&info->lock);
1189                 return ret;
1190         } else if (dmatest_run) {
1191                 if (!is_threaded_test_pending(info)) {
1192                         /*
1193                          * We have nothing to run. This can be due to:
1194                          */
1195                         ret = info->last_error;
1196                         if (ret) {
1197                                 /* 1) Misconfiguration */
1198                                 pr_err("Channel misconfigured, can't continue\n");
1199                                 mutex_unlock(&info->lock);
1200                                 return ret;
1201                         } else {
1202                                 /* 2) We rely on defaults */
1203                                 pr_info("No channels configured, continue with any\n");
1204                                 if (!is_threaded_test_run(info))
1205                                         stop_threaded_test(info);
1206                                 add_threaded_test(info);
1207                         }
1208                 }
1209                 start_threaded_tests(info);
1210         } else {
1211                 stop_threaded_test(info);
1212         }
1213
1214         mutex_unlock(&info->lock);
1215
1216         return ret;
1217 }
1218
1219 static int dmatest_chan_set(const char *val, const struct kernel_param *kp)
1220 {
1221         struct dmatest_info *info = &test_info;
1222         struct dmatest_chan *dtc;
1223         char chan_reset_val[20];
1224         int ret;
1225
1226         mutex_lock(&info->lock);
1227         ret = param_set_copystring(val, kp);
1228         if (ret) {
1229                 mutex_unlock(&info->lock);
1230                 return ret;
1231         }
1232         /*Clear any previously run threads */
1233         if (!is_threaded_test_run(info) && !is_threaded_test_pending(info))
1234                 stop_threaded_test(info);
1235         /* Reject channels that are already registered */
1236         if (is_threaded_test_pending(info)) {
1237                 list_for_each_entry(dtc, &info->channels, node) {
1238                         if (strcmp(dma_chan_name(dtc->chan),
1239                                    strim(test_channel)) == 0) {
1240                                 dtc = list_last_entry(&info->channels,
1241                                                       struct dmatest_chan,
1242                                                       node);
1243                                 strlcpy(chan_reset_val,
1244                                         dma_chan_name(dtc->chan),
1245                                         sizeof(chan_reset_val));
1246                                 ret = -EBUSY;
1247                                 goto add_chan_err;
1248                         }
1249                 }
1250         }
1251
1252         add_threaded_test(info);
1253
1254         /* Check if channel was added successfully */
1255         if (!list_empty(&info->channels)) {
1256                 /*
1257                  * if new channel was not successfully added, revert the
1258                  * "test_channel" string to the name of the last successfully
1259                  * added channel. exception for when users issues empty string
1260                  * to channel parameter.
1261                  */
1262                 dtc = list_last_entry(&info->channels, struct dmatest_chan, node);
1263                 if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0)
1264                     && (strcmp("", strim(test_channel)) != 0)) {
1265                         ret = -EINVAL;
1266                         strlcpy(chan_reset_val, dma_chan_name(dtc->chan),
1267                                 sizeof(chan_reset_val));
1268                         goto add_chan_err;
1269                 }
1270
1271         } else {
1272                 /* Clear test_channel if no channels were added successfully */
1273                 strlcpy(chan_reset_val, "", sizeof(chan_reset_val));
1274                 ret = -EBUSY;
1275                 goto add_chan_err;
1276         }
1277
1278         info->last_error = ret;
1279         mutex_unlock(&info->lock);
1280
1281         return ret;
1282
1283 add_chan_err:
1284         param_set_copystring(chan_reset_val, kp);
1285         info->last_error = ret;
1286         mutex_unlock(&info->lock);
1287
1288         return ret;
1289 }
1290
1291 static int dmatest_chan_get(char *val, const struct kernel_param *kp)
1292 {
1293         struct dmatest_info *info = &test_info;
1294
1295         mutex_lock(&info->lock);
1296         if (!is_threaded_test_run(info) && !is_threaded_test_pending(info)) {
1297                 stop_threaded_test(info);
1298                 strlcpy(test_channel, "", sizeof(test_channel));
1299         }
1300         mutex_unlock(&info->lock);
1301
1302         return param_get_string(val, kp);
1303 }
1304
1305 static int dmatest_test_list_get(char *val, const struct kernel_param *kp)
1306 {
1307         struct dmatest_info *info = &test_info;
1308         struct dmatest_chan *dtc;
1309         unsigned int thread_count = 0;
1310
1311         list_for_each_entry(dtc, &info->channels, node) {
1312                 struct dmatest_thread *thread;
1313
1314                 thread_count = 0;
1315                 list_for_each_entry(thread, &dtc->threads, node) {
1316                         thread_count++;
1317                 }
1318                 pr_info("%u threads using %s\n",
1319                         thread_count, dma_chan_name(dtc->chan));
1320         }
1321
1322         return 0;
1323 }
1324
1325 static int __init dmatest_init(void)
1326 {
1327         struct dmatest_info *info = &test_info;
1328         struct dmatest_params *params = &info->params;
1329
1330         if (dmatest_run) {
1331                 mutex_lock(&info->lock);
1332                 add_threaded_test(info);
1333                 run_pending_tests(info);
1334                 mutex_unlock(&info->lock);
1335         }
1336
1337         if (params->iterations && wait)
1338                 wait_event(thread_wait, !is_threaded_test_run(info));
1339
1340         /* module parameters are stable, inittime tests are started,
1341          * let userspace take over 'run' control
1342          */
1343         info->did_init = true;
1344
1345         return 0;
1346 }
1347 /* when compiled-in wait for drivers to load first */
1348 late_initcall(dmatest_init);
1349
1350 static void __exit dmatest_exit(void)
1351 {
1352         struct dmatest_info *info = &test_info;
1353
1354         mutex_lock(&info->lock);
1355         stop_threaded_test(info);
1356         mutex_unlock(&info->lock);
1357 }
1358 module_exit(dmatest_exit);
1359
1360 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1361 MODULE_LICENSE("GPL v2");