1 // SPDX-License-Identifier: GPL-2.0
3 * KCSAN test with various race scenarious to test runtime behaviour. Since the
4 * interface with which KCSAN's reports are obtained is via the console, this is
5 * the output we should verify. For each test case checks the presence (or
6 * absence) of generated reports. Relies on 'console' tracepoint to capture
7 * reports as they appear in the kernel log.
9 * Makes use of KUnit for test organization, and the Torture framework for test
12 * Copyright (C) 2020, Google LLC.
13 * Author: Marco Elver <elver@google.com>
16 #define pr_fmt(fmt) "kcsan_test: " fmt
18 #include <kunit/test.h>
19 #include <linux/jiffies.h>
20 #include <linux/kcsan-checks.h>
21 #include <linux/kernel.h>
22 #include <linux/sched.h>
23 #include <linux/seqlock.h>
24 #include <linux/spinlock.h>
25 #include <linux/string.h>
26 #include <linux/timer.h>
27 #include <linux/torture.h>
28 #include <linux/tracepoint.h>
29 #include <linux/types.h>
30 #include <trace/events/printk.h>
32 #ifdef CONFIG_CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE
33 #define __KCSAN_ACCESS_RW(alt) (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
35 #define __KCSAN_ACCESS_RW(alt) (alt)
38 /* Points to current test-case memory access "kernels". */
39 static void (*access_kernels[2])(void);
41 static struct task_struct **threads; /* Lists of threads. */
42 static unsigned long end_time; /* End time of test. */
44 /* Report as observed from console. */
50 .lock = __SPIN_LOCK_UNLOCKED(observed.lock),
53 /* Setup test checking loop. */
54 static __no_kcsan inline void
55 begin_test_checks(void (*func1)(void), void (*func2)(void))
57 kcsan_disable_current();
60 * Require at least as long as KCSAN_REPORT_ONCE_IN_MS, to ensure at
61 * least one race is reported.
63 end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500);
65 /* Signal start; release potential initialization of shared data. */
66 smp_store_release(&access_kernels[0], func1);
67 smp_store_release(&access_kernels[1], func2);
70 /* End test checking loop. */
71 static __no_kcsan inline bool
72 end_test_checks(bool stop)
74 if (!stop && time_before(jiffies, end_time)) {
75 /* Continue checking */
80 kcsan_enable_current();
85 * Probe for console output: checks if a race was reported, and obtains observed
89 static void probe_console(void *ignore, const char *buf, size_t len)
95 * Note that KCSAN reports under a global lock, so we do not risk the
96 * possibility of having multiple reports interleaved. If that were the
97 * case, we'd expect tests to fail.
100 spin_lock_irqsave(&observed.lock, flags);
101 nlines = observed.nlines;
103 if (strnstr(buf, "BUG: KCSAN: ", len) && strnstr(buf, "test_", len)) {
105 * KCSAN report and related to the test.
107 * The provided @buf is not NUL-terminated; copy no more than
108 * @len bytes and let strscpy() add the missing NUL-terminator.
110 strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0])));
112 } else if ((nlines == 1 || nlines == 2) && strnstr(buf, "bytes by", len)) {
113 strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0])));
115 if (strnstr(buf, "race at unknown origin", len)) {
116 if (WARN_ON(nlines != 2))
119 /* No second line of interest. */
120 strcpy(observed.lines[nlines++], "<none>");
125 WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */
126 spin_unlock_irqrestore(&observed.lock, flags);
129 /* Check if a report related to the test exists. */
131 static bool report_available(void)
133 return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines);
136 /* Report information we expect in a report. */
137 struct expect_report {
138 /* Access information of both accesses. */
140 void *fn; /* Function pointer to expected function of top frame. */
141 void *addr; /* Address of access; unchecked if NULL. */
142 size_t size; /* Size of access; unchecked if @addr is NULL. */
143 int type; /* Access type, see KCSAN_ACCESS definitions. */
147 /* Check observed report matches information in @r. */
149 static bool report_matches(const struct expect_report *r)
151 const bool is_assert = (r->access[0].type | r->access[1].type) & KCSAN_ACCESS_ASSERT;
154 typeof(*observed.lines) *expect;
159 /* Doubled-checked locking. */
160 if (!report_available())
163 expect = kmalloc(sizeof(observed.lines), GFP_KERNEL);
164 if (WARN_ON(!expect))
167 /* Generate expected report contents. */
171 end = &expect[0][sizeof(expect[0]) - 1];
172 cur += scnprintf(cur, end - cur, "BUG: KCSAN: %s in ",
173 is_assert ? "assert: race" : "data-race");
174 if (r->access[1].fn) {
178 /* Expect lexographically sorted function names in title. */
179 scnprintf(tmp[0], sizeof(tmp[0]), "%pS", r->access[0].fn);
180 scnprintf(tmp[1], sizeof(tmp[1]), "%pS", r->access[1].fn);
181 cmp = strcmp(tmp[0], tmp[1]);
182 cur += scnprintf(cur, end - cur, "%ps / %ps",
183 cmp < 0 ? r->access[0].fn : r->access[1].fn,
184 cmp < 0 ? r->access[1].fn : r->access[0].fn);
186 scnprintf(cur, end - cur, "%pS", r->access[0].fn);
187 /* The exact offset won't match, remove it. */
188 cur = strchr(expect[0], '+');
195 end = &expect[1][sizeof(expect[1]) - 1];
196 if (!r->access[1].fn)
197 cur += scnprintf(cur, end - cur, "race at unknown origin, with ");
200 for (i = 0; i < 2; ++i) {
201 const int ty = r->access[i].type;
202 const char *const access_type =
203 (ty & KCSAN_ACCESS_ASSERT) ?
204 ((ty & KCSAN_ACCESS_WRITE) ?
205 "assert no accesses" :
206 "assert no writes") :
207 ((ty & KCSAN_ACCESS_WRITE) ?
208 ((ty & KCSAN_ACCESS_COMPOUND) ?
212 const char *const access_type_aux =
213 (ty & KCSAN_ACCESS_ATOMIC) ?
215 ((ty & KCSAN_ACCESS_SCOPED) ? " (scoped)" : "");
220 end = &expect[2][sizeof(expect[2]) - 1];
222 if (!r->access[1].fn) {
223 /* Dummy string if no second access is available. */
224 strcpy(cur, "<none>");
229 cur += scnprintf(cur, end - cur, "%s%s to ", access_type,
232 if (r->access[i].addr) /* Address is optional. */
233 cur += scnprintf(cur, end - cur, "0x%px of %zu bytes",
234 r->access[i].addr, r->access[i].size);
237 spin_lock_irqsave(&observed.lock, flags);
238 if (!report_available())
239 goto out; /* A new report is being captured. */
241 /* Finally match expected output to what we actually observed. */
242 ret = strstr(observed.lines[0], expect[0]) &&
243 /* Access info may appear in any order. */
244 ((strstr(observed.lines[1], expect[1]) &&
245 strstr(observed.lines[2], expect[2])) ||
246 (strstr(observed.lines[1], expect[2]) &&
247 strstr(observed.lines[2], expect[1])));
249 spin_unlock_irqrestore(&observed.lock, flags);
254 /* ===== Test kernels ===== */
256 static long test_sink;
257 static long test_var;
258 /* @test_array should be large enough to fall into multiple watchpoint slots. */
259 static long test_array[3 * PAGE_SIZE / sizeof(long)];
263 static DEFINE_SEQLOCK(test_seqlock);
266 * Helper to avoid compiler optimizing out reads, and to generate source values
270 static noinline void sink_value(long v) { WRITE_ONCE(test_sink, v); }
272 static noinline void test_kernel_read(void) { sink_value(test_var); }
274 static noinline void test_kernel_write(void)
276 test_var = READ_ONCE_NOCHECK(test_sink) + 1;
279 static noinline void test_kernel_write_nochange(void) { test_var = 42; }
281 /* Suffixed by value-change exception filter. */
282 static noinline void test_kernel_write_nochange_rcu(void) { test_var = 42; }
284 static noinline void test_kernel_read_atomic(void)
286 sink_value(READ_ONCE(test_var));
289 static noinline void test_kernel_write_atomic(void)
291 WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1);
294 static noinline void test_kernel_atomic_rmw(void)
296 /* Use builtin, so we can set up the "bad" atomic/non-atomic scenario. */
297 __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED);
301 static noinline void test_kernel_write_uninstrumented(void) { test_var++; }
303 static noinline void test_kernel_data_race(void) { data_race(test_var++); }
305 static noinline void test_kernel_assert_writer(void)
307 ASSERT_EXCLUSIVE_WRITER(test_var);
310 static noinline void test_kernel_assert_access(void)
312 ASSERT_EXCLUSIVE_ACCESS(test_var);
315 #define TEST_CHANGE_BITS 0xff00ff00
317 static noinline void test_kernel_change_bits(void)
319 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {
321 * Avoid race of unknown origin for this test, just pretend they
324 kcsan_nestable_atomic_begin();
325 test_var ^= TEST_CHANGE_BITS;
326 kcsan_nestable_atomic_end();
328 WRITE_ONCE(test_var, READ_ONCE(test_var) ^ TEST_CHANGE_BITS);
331 static noinline void test_kernel_assert_bits_change(void)
333 ASSERT_EXCLUSIVE_BITS(test_var, TEST_CHANGE_BITS);
336 static noinline void test_kernel_assert_bits_nochange(void)
338 ASSERT_EXCLUSIVE_BITS(test_var, ~TEST_CHANGE_BITS);
341 /* To check that scoped assertions do trigger anywhere in scope. */
342 static noinline void test_enter_scope(void)
346 /* Unrelated accesses to scoped assert. */
347 READ_ONCE(test_sink);
348 kcsan_check_read(&x, sizeof(x));
351 static noinline void test_kernel_assert_writer_scoped(void)
353 ASSERT_EXCLUSIVE_WRITER_SCOPED(test_var);
357 static noinline void test_kernel_assert_access_scoped(void)
359 ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_var);
363 static noinline void test_kernel_rmw_array(void)
367 for (i = 0; i < ARRAY_SIZE(test_array); ++i)
371 static noinline void test_kernel_write_struct(void)
373 kcsan_check_write(&test_struct, sizeof(test_struct));
374 kcsan_disable_current();
375 test_struct.val[3]++; /* induce value change */
376 kcsan_enable_current();
379 static noinline void test_kernel_write_struct_part(void)
381 test_struct.val[3] = 42;
384 static noinline void test_kernel_read_struct_zero_size(void)
386 kcsan_check_read(&test_struct.val[3], 0);
389 static noinline void test_kernel_jiffies_reader(void)
391 sink_value((long)jiffies);
394 static noinline void test_kernel_seqlock_reader(void)
399 seq = read_seqbegin(&test_seqlock);
400 sink_value(test_var);
401 } while (read_seqretry(&test_seqlock, seq));
404 static noinline void test_kernel_seqlock_writer(void)
408 write_seqlock_irqsave(&test_seqlock, flags);
410 write_sequnlock_irqrestore(&test_seqlock, flags);
413 static noinline void test_kernel_atomic_builtins(void)
416 * Generate concurrent accesses, expecting no reports, ensuring KCSAN
417 * treats builtin atomics as actually atomic.
419 __atomic_load_n(&test_var, __ATOMIC_RELAXED);
422 static noinline void test_kernel_xor_1bit(void)
424 /* Do not report data races between the read-writes. */
425 kcsan_nestable_atomic_begin();
427 kcsan_nestable_atomic_end();
430 /* ===== Test cases ===== */
432 /* Simple test with normal data race. */
434 static void test_basic(struct kunit *test)
436 const struct expect_report expect = {
438 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
439 { test_kernel_read, &test_var, sizeof(test_var), 0 },
442 static const struct expect_report never = {
444 { test_kernel_read, &test_var, sizeof(test_var), 0 },
445 { test_kernel_read, &test_var, sizeof(test_var), 0 },
448 bool match_expect = false;
449 bool match_never = false;
451 begin_test_checks(test_kernel_write, test_kernel_read);
453 match_expect |= report_matches(&expect);
454 match_never = report_matches(&never);
455 } while (!end_test_checks(match_never));
456 KUNIT_EXPECT_TRUE(test, match_expect);
457 KUNIT_EXPECT_FALSE(test, match_never);
461 * Stress KCSAN with lots of concurrent races on different addresses until
465 static void test_concurrent_races(struct kunit *test)
467 const struct expect_report expect = {
469 /* NULL will match any address. */
470 { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
471 { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(0) },
474 static const struct expect_report never = {
476 { test_kernel_rmw_array, NULL, 0, 0 },
477 { test_kernel_rmw_array, NULL, 0, 0 },
480 bool match_expect = false;
481 bool match_never = false;
483 begin_test_checks(test_kernel_rmw_array, test_kernel_rmw_array);
485 match_expect |= report_matches(&expect);
486 match_never |= report_matches(&never);
487 } while (!end_test_checks(false));
488 KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check matches exist. */
489 KUNIT_EXPECT_FALSE(test, match_never);
492 /* Test the KCSAN_REPORT_VALUE_CHANGE_ONLY option. */
494 static void test_novalue_change(struct kunit *test)
496 const struct expect_report expect = {
498 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
499 { test_kernel_read, &test_var, sizeof(test_var), 0 },
502 bool match_expect = false;
504 begin_test_checks(test_kernel_write_nochange, test_kernel_read);
506 match_expect = report_matches(&expect);
507 } while (!end_test_checks(match_expect));
508 if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY))
509 KUNIT_EXPECT_FALSE(test, match_expect);
511 KUNIT_EXPECT_TRUE(test, match_expect);
515 * Test that the rules where the KCSAN_REPORT_VALUE_CHANGE_ONLY option should
519 static void test_novalue_change_exception(struct kunit *test)
521 const struct expect_report expect = {
523 { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
524 { test_kernel_read, &test_var, sizeof(test_var), 0 },
527 bool match_expect = false;
529 begin_test_checks(test_kernel_write_nochange_rcu, test_kernel_read);
531 match_expect = report_matches(&expect);
532 } while (!end_test_checks(match_expect));
533 KUNIT_EXPECT_TRUE(test, match_expect);
536 /* Test that data races of unknown origin are reported. */
538 static void test_unknown_origin(struct kunit *test)
540 const struct expect_report expect = {
542 { test_kernel_read, &test_var, sizeof(test_var), 0 },
546 bool match_expect = false;
548 begin_test_checks(test_kernel_write_uninstrumented, test_kernel_read);
550 match_expect = report_matches(&expect);
551 } while (!end_test_checks(match_expect));
552 if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN))
553 KUNIT_EXPECT_TRUE(test, match_expect);
555 KUNIT_EXPECT_FALSE(test, match_expect);
558 /* Test KCSAN_ASSUME_PLAIN_WRITES_ATOMIC if it is selected. */
560 static void test_write_write_assume_atomic(struct kunit *test)
562 const struct expect_report expect = {
564 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
565 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
568 bool match_expect = false;
570 begin_test_checks(test_kernel_write, test_kernel_write);
572 sink_value(READ_ONCE(test_var)); /* induce value-change */
573 match_expect = report_matches(&expect);
574 } while (!end_test_checks(match_expect));
575 if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC))
576 KUNIT_EXPECT_FALSE(test, match_expect);
578 KUNIT_EXPECT_TRUE(test, match_expect);
582 * Test that data races with writes larger than word-size are always reported,
583 * even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
586 static void test_write_write_struct(struct kunit *test)
588 const struct expect_report expect = {
590 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
591 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
594 bool match_expect = false;
596 begin_test_checks(test_kernel_write_struct, test_kernel_write_struct);
598 match_expect = report_matches(&expect);
599 } while (!end_test_checks(match_expect));
600 KUNIT_EXPECT_TRUE(test, match_expect);
604 * Test that data races where only one write is larger than word-size are always
605 * reported, even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
608 static void test_write_write_struct_part(struct kunit *test)
610 const struct expect_report expect = {
612 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
613 { test_kernel_write_struct_part, &test_struct.val[3], sizeof(test_struct.val[3]), KCSAN_ACCESS_WRITE },
616 bool match_expect = false;
618 begin_test_checks(test_kernel_write_struct, test_kernel_write_struct_part);
620 match_expect = report_matches(&expect);
621 } while (!end_test_checks(match_expect));
622 KUNIT_EXPECT_TRUE(test, match_expect);
625 /* Test that races with atomic accesses never result in reports. */
627 static void test_read_atomic_write_atomic(struct kunit *test)
629 bool match_never = false;
631 begin_test_checks(test_kernel_read_atomic, test_kernel_write_atomic);
633 match_never = report_available();
634 } while (!end_test_checks(match_never));
635 KUNIT_EXPECT_FALSE(test, match_never);
638 /* Test that a race with an atomic and plain access result in reports. */
640 static void test_read_plain_atomic_write(struct kunit *test)
642 const struct expect_report expect = {
644 { test_kernel_read, &test_var, sizeof(test_var), 0 },
645 { test_kernel_write_atomic, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
648 bool match_expect = false;
650 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS))
653 begin_test_checks(test_kernel_read, test_kernel_write_atomic);
655 match_expect = report_matches(&expect);
656 } while (!end_test_checks(match_expect));
657 KUNIT_EXPECT_TRUE(test, match_expect);
660 /* Test that atomic RMWs generate correct report. */
662 static void test_read_plain_atomic_rmw(struct kunit *test)
664 const struct expect_report expect = {
666 { test_kernel_read, &test_var, sizeof(test_var), 0 },
667 { test_kernel_atomic_rmw, &test_var, sizeof(test_var),
668 KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
671 bool match_expect = false;
673 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS))
676 begin_test_checks(test_kernel_read, test_kernel_atomic_rmw);
678 match_expect = report_matches(&expect);
679 } while (!end_test_checks(match_expect));
680 KUNIT_EXPECT_TRUE(test, match_expect);
683 /* Zero-sized accesses should never cause data race reports. */
685 static void test_zero_size_access(struct kunit *test)
687 const struct expect_report expect = {
689 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
690 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
693 const struct expect_report never = {
695 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
696 { test_kernel_read_struct_zero_size, &test_struct.val[3], 0, 0 },
699 bool match_expect = false;
700 bool match_never = false;
702 begin_test_checks(test_kernel_write_struct, test_kernel_read_struct_zero_size);
704 match_expect |= report_matches(&expect);
705 match_never = report_matches(&never);
706 } while (!end_test_checks(match_never));
707 KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check. */
708 KUNIT_EXPECT_FALSE(test, match_never);
711 /* Test the data_race() macro. */
713 static void test_data_race(struct kunit *test)
715 bool match_never = false;
717 begin_test_checks(test_kernel_data_race, test_kernel_data_race);
719 match_never = report_available();
720 } while (!end_test_checks(match_never));
721 KUNIT_EXPECT_FALSE(test, match_never);
725 static void test_assert_exclusive_writer(struct kunit *test)
727 const struct expect_report expect = {
729 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
730 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
733 bool match_expect = false;
735 begin_test_checks(test_kernel_assert_writer, test_kernel_write_nochange);
737 match_expect = report_matches(&expect);
738 } while (!end_test_checks(match_expect));
739 KUNIT_EXPECT_TRUE(test, match_expect);
743 static void test_assert_exclusive_access(struct kunit *test)
745 const struct expect_report expect = {
747 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
748 { test_kernel_read, &test_var, sizeof(test_var), 0 },
751 bool match_expect = false;
753 begin_test_checks(test_kernel_assert_access, test_kernel_read);
755 match_expect = report_matches(&expect);
756 } while (!end_test_checks(match_expect));
757 KUNIT_EXPECT_TRUE(test, match_expect);
761 static void test_assert_exclusive_access_writer(struct kunit *test)
763 const struct expect_report expect_access_writer = {
765 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
766 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
769 const struct expect_report expect_access_access = {
771 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
772 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
775 const struct expect_report never = {
777 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
778 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
781 bool match_expect_access_writer = false;
782 bool match_expect_access_access = false;
783 bool match_never = false;
785 begin_test_checks(test_kernel_assert_access, test_kernel_assert_writer);
787 match_expect_access_writer |= report_matches(&expect_access_writer);
788 match_expect_access_access |= report_matches(&expect_access_access);
789 match_never |= report_matches(&never);
790 } while (!end_test_checks(match_never));
791 KUNIT_EXPECT_TRUE(test, match_expect_access_writer);
792 KUNIT_EXPECT_TRUE(test, match_expect_access_access);
793 KUNIT_EXPECT_FALSE(test, match_never);
797 static void test_assert_exclusive_bits_change(struct kunit *test)
799 const struct expect_report expect = {
801 { test_kernel_assert_bits_change, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
802 { test_kernel_change_bits, &test_var, sizeof(test_var),
803 KCSAN_ACCESS_WRITE | (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) ? 0 : KCSAN_ACCESS_ATOMIC) },
806 bool match_expect = false;
808 begin_test_checks(test_kernel_assert_bits_change, test_kernel_change_bits);
810 match_expect = report_matches(&expect);
811 } while (!end_test_checks(match_expect));
812 KUNIT_EXPECT_TRUE(test, match_expect);
816 static void test_assert_exclusive_bits_nochange(struct kunit *test)
818 bool match_never = false;
820 begin_test_checks(test_kernel_assert_bits_nochange, test_kernel_change_bits);
822 match_never = report_available();
823 } while (!end_test_checks(match_never));
824 KUNIT_EXPECT_FALSE(test, match_never);
828 static void test_assert_exclusive_writer_scoped(struct kunit *test)
830 const struct expect_report expect_start = {
832 { test_kernel_assert_writer_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
833 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
836 const struct expect_report expect_anywhere = {
838 { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
839 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
842 bool match_expect_start = false;
843 bool match_expect_anywhere = false;
845 begin_test_checks(test_kernel_assert_writer_scoped, test_kernel_write_nochange);
847 match_expect_start |= report_matches(&expect_start);
848 match_expect_anywhere |= report_matches(&expect_anywhere);
849 } while (!end_test_checks(match_expect_start && match_expect_anywhere));
850 KUNIT_EXPECT_TRUE(test, match_expect_start);
851 KUNIT_EXPECT_TRUE(test, match_expect_anywhere);
855 static void test_assert_exclusive_access_scoped(struct kunit *test)
857 const struct expect_report expect_start1 = {
859 { test_kernel_assert_access_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
860 { test_kernel_read, &test_var, sizeof(test_var), 0 },
863 const struct expect_report expect_start2 = {
864 .access = { expect_start1.access[0], expect_start1.access[0] },
866 const struct expect_report expect_inscope = {
868 { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
869 { test_kernel_read, &test_var, sizeof(test_var), 0 },
872 bool match_expect_start = false;
873 bool match_expect_inscope = false;
875 begin_test_checks(test_kernel_assert_access_scoped, test_kernel_read);
876 end_time += msecs_to_jiffies(1000); /* This test requires a bit more time. */
878 match_expect_start |= report_matches(&expect_start1) || report_matches(&expect_start2);
879 match_expect_inscope |= report_matches(&expect_inscope);
880 } while (!end_test_checks(match_expect_start && match_expect_inscope));
881 KUNIT_EXPECT_TRUE(test, match_expect_start);
882 KUNIT_EXPECT_TRUE(test, match_expect_inscope);
886 * jiffies is special (declared to be volatile) and its accesses are typically
887 * not marked; this test ensures that the compiler nor KCSAN gets confused about
888 * jiffies's declaration on different architectures.
891 static void test_jiffies_noreport(struct kunit *test)
893 bool match_never = false;
895 begin_test_checks(test_kernel_jiffies_reader, test_kernel_jiffies_reader);
897 match_never = report_available();
898 } while (!end_test_checks(match_never));
899 KUNIT_EXPECT_FALSE(test, match_never);
902 /* Test that racing accesses in seqlock critical sections are not reported. */
904 static void test_seqlock_noreport(struct kunit *test)
906 bool match_never = false;
908 begin_test_checks(test_kernel_seqlock_reader, test_kernel_seqlock_writer);
910 match_never = report_available();
911 } while (!end_test_checks(match_never));
912 KUNIT_EXPECT_FALSE(test, match_never);
916 * Test atomic builtins work and required instrumentation functions exist. We
917 * also test that KCSAN understands they're atomic by racing with them via
918 * test_kernel_atomic_builtins(), and expect no reports.
920 * The atomic builtins _SHOULD NOT_ be used in normal kernel code!
922 static void test_atomic_builtins(struct kunit *test)
924 bool match_never = false;
926 begin_test_checks(test_kernel_atomic_builtins, test_kernel_atomic_builtins);
930 kcsan_enable_current();
932 __atomic_store_n(&test_var, 42L, __ATOMIC_RELAXED);
933 KUNIT_EXPECT_EQ(test, 42L, __atomic_load_n(&test_var, __ATOMIC_RELAXED));
935 KUNIT_EXPECT_EQ(test, 42L, __atomic_exchange_n(&test_var, 20, __ATOMIC_RELAXED));
936 KUNIT_EXPECT_EQ(test, 20L, test_var);
939 KUNIT_EXPECT_TRUE(test, __atomic_compare_exchange_n(&test_var, &tmp, 30L,
942 KUNIT_EXPECT_EQ(test, tmp, 20L);
943 KUNIT_EXPECT_EQ(test, test_var, 30L);
944 KUNIT_EXPECT_FALSE(test, __atomic_compare_exchange_n(&test_var, &tmp, 40L,
947 KUNIT_EXPECT_EQ(test, tmp, 30L);
948 KUNIT_EXPECT_EQ(test, test_var, 30L);
950 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED));
951 KUNIT_EXPECT_EQ(test, 31L, __atomic_fetch_sub(&test_var, 1, __ATOMIC_RELAXED));
952 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_and(&test_var, 0xf, __ATOMIC_RELAXED));
953 KUNIT_EXPECT_EQ(test, 14L, __atomic_fetch_xor(&test_var, 0xf, __ATOMIC_RELAXED));
954 KUNIT_EXPECT_EQ(test, 1L, __atomic_fetch_or(&test_var, 0xf0, __ATOMIC_RELAXED));
955 KUNIT_EXPECT_EQ(test, 241L, __atomic_fetch_nand(&test_var, 0xf, __ATOMIC_RELAXED));
956 KUNIT_EXPECT_EQ(test, -2L, test_var);
958 __atomic_thread_fence(__ATOMIC_SEQ_CST);
959 __atomic_signal_fence(__ATOMIC_SEQ_CST);
961 kcsan_disable_current();
963 match_never = report_available();
964 } while (!end_test_checks(match_never));
965 KUNIT_EXPECT_FALSE(test, match_never);
969 static void test_1bit_value_change(struct kunit *test)
971 const struct expect_report expect = {
973 { test_kernel_read, &test_var, sizeof(test_var), 0 },
974 { test_kernel_xor_1bit, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
979 begin_test_checks(test_kernel_read, test_kernel_xor_1bit);
981 match = IS_ENABLED(CONFIG_KCSAN_PERMISSIVE)
983 : report_matches(&expect);
984 } while (!end_test_checks(match));
985 if (IS_ENABLED(CONFIG_KCSAN_PERMISSIVE))
986 KUNIT_EXPECT_FALSE(test, match);
988 KUNIT_EXPECT_TRUE(test, match);
992 * Generate thread counts for all test cases. Values generated are in interval
993 * [2, 5] followed by exponentially increasing thread counts from 8 to 32.
995 * The thread counts are chosen to cover potentially interesting boundaries and
996 * corner cases (2 to 5), and then stress the system with larger counts.
998 static const void *nthreads_gen_params(const void *prev, char *desc)
1000 long nthreads = (long)prev;
1002 if (nthreads < 0 || nthreads >= 32)
1003 nthreads = 0; /* stop */
1005 nthreads = 2; /* initial value */
1006 else if (nthreads < 5)
1008 else if (nthreads == 5)
1013 if (!IS_ENABLED(CONFIG_PREEMPT) || !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
1015 * Without any preemption, keep 2 CPUs free for other tasks, one
1016 * of which is the main test case function checking for
1017 * completion or failure.
1019 const long min_unused_cpus = IS_ENABLED(CONFIG_PREEMPT_NONE) ? 2 : 0;
1020 const long min_required_cpus = 2 + min_unused_cpus;
1022 if (num_online_cpus() < min_required_cpus) {
1023 pr_err_once("Too few online CPUs (%u < %ld) for test\n",
1024 num_online_cpus(), min_required_cpus);
1026 } else if (nthreads >= num_online_cpus() - min_unused_cpus) {
1027 /* Use negative value to indicate last param. */
1028 nthreads = -(num_online_cpus() - min_unused_cpus);
1029 pr_warn_once("Limiting number of threads to %ld (only %d online CPUs)\n",
1030 -nthreads, num_online_cpus());
1034 snprintf(desc, KUNIT_PARAM_DESC_SIZE, "threads=%ld", abs(nthreads));
1035 return (void *)nthreads;
1038 #define KCSAN_KUNIT_CASE(test_name) KUNIT_CASE_PARAM(test_name, nthreads_gen_params)
1039 static struct kunit_case kcsan_test_cases[] = {
1040 KCSAN_KUNIT_CASE(test_basic),
1041 KCSAN_KUNIT_CASE(test_concurrent_races),
1042 KCSAN_KUNIT_CASE(test_novalue_change),
1043 KCSAN_KUNIT_CASE(test_novalue_change_exception),
1044 KCSAN_KUNIT_CASE(test_unknown_origin),
1045 KCSAN_KUNIT_CASE(test_write_write_assume_atomic),
1046 KCSAN_KUNIT_CASE(test_write_write_struct),
1047 KCSAN_KUNIT_CASE(test_write_write_struct_part),
1048 KCSAN_KUNIT_CASE(test_read_atomic_write_atomic),
1049 KCSAN_KUNIT_CASE(test_read_plain_atomic_write),
1050 KCSAN_KUNIT_CASE(test_read_plain_atomic_rmw),
1051 KCSAN_KUNIT_CASE(test_zero_size_access),
1052 KCSAN_KUNIT_CASE(test_data_race),
1053 KCSAN_KUNIT_CASE(test_assert_exclusive_writer),
1054 KCSAN_KUNIT_CASE(test_assert_exclusive_access),
1055 KCSAN_KUNIT_CASE(test_assert_exclusive_access_writer),
1056 KCSAN_KUNIT_CASE(test_assert_exclusive_bits_change),
1057 KCSAN_KUNIT_CASE(test_assert_exclusive_bits_nochange),
1058 KCSAN_KUNIT_CASE(test_assert_exclusive_writer_scoped),
1059 KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped),
1060 KCSAN_KUNIT_CASE(test_jiffies_noreport),
1061 KCSAN_KUNIT_CASE(test_seqlock_noreport),
1062 KCSAN_KUNIT_CASE(test_atomic_builtins),
1063 KCSAN_KUNIT_CASE(test_1bit_value_change),
1067 /* ===== End test cases ===== */
1069 /* Concurrent accesses from interrupts. */
1071 static void access_thread_timer(struct timer_list *timer)
1073 static atomic_t cnt = ATOMIC_INIT(0);
1077 idx = (unsigned int)atomic_inc_return(&cnt) % ARRAY_SIZE(access_kernels);
1078 /* Acquire potential initialization. */
1079 func = smp_load_acquire(&access_kernels[idx]);
1084 /* The main loop for each thread. */
1086 static int access_thread(void *arg)
1088 struct timer_list timer;
1089 unsigned int cnt = 0;
1093 timer_setup_on_stack(&timer, access_thread_timer, 0);
1097 if (!timer_pending(&timer))
1098 mod_timer(&timer, jiffies + 1);
1100 /* Iterate through all kernels. */
1101 idx = cnt++ % ARRAY_SIZE(access_kernels);
1102 /* Acquire potential initialization. */
1103 func = smp_load_acquire(&access_kernels[idx]);
1107 } while (!torture_must_stop());
1108 del_timer_sync(&timer);
1109 destroy_timer_on_stack(&timer);
1111 torture_kthread_stopping("access_thread");
1116 static int test_init(struct kunit *test)
1118 unsigned long flags;
1122 spin_lock_irqsave(&observed.lock, flags);
1123 for (i = 0; i < ARRAY_SIZE(observed.lines); ++i)
1124 observed.lines[i][0] = '\0';
1125 observed.nlines = 0;
1126 spin_unlock_irqrestore(&observed.lock, flags);
1128 if (!torture_init_begin((char *)test->name, 1))
1131 if (WARN_ON(threads))
1134 for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) {
1135 if (WARN_ON(access_kernels[i]))
1139 nthreads = abs((long)test->param_value);
1140 if (WARN_ON(!nthreads))
1143 threads = kcalloc(nthreads + 1, sizeof(struct task_struct *), GFP_KERNEL);
1144 if (WARN_ON(!threads))
1147 threads[nthreads] = NULL;
1148 for (i = 0; i < nthreads; ++i) {
1149 if (torture_create_kthread(access_thread, NULL, threads[i]))
1165 static void test_exit(struct kunit *test)
1167 struct task_struct **stop_thread;
1170 if (torture_cleanup_begin())
1173 for (i = 0; i < ARRAY_SIZE(access_kernels); ++i)
1174 WRITE_ONCE(access_kernels[i], NULL);
1177 for (stop_thread = threads; *stop_thread; stop_thread++)
1178 torture_stop_kthread(reader_thread, *stop_thread);
1184 torture_cleanup_end();
1187 static struct kunit_suite kcsan_test_suite = {
1189 .test_cases = kcsan_test_cases,
1193 static struct kunit_suite *kcsan_test_suites[] = { &kcsan_test_suite, NULL };
1196 static void register_tracepoints(struct tracepoint *tp, void *ignore)
1198 check_trace_callback_type_console(probe_console);
1199 if (!strcmp(tp->name, "console"))
1200 WARN_ON(tracepoint_probe_register(tp, probe_console, NULL));
1204 static void unregister_tracepoints(struct tracepoint *tp, void *ignore)
1206 if (!strcmp(tp->name, "console"))
1207 tracepoint_probe_unregister(tp, probe_console, NULL);
1211 * We only want to do tracepoints setup and teardown once, therefore we have to
1212 * customize the init and exit functions and cannot rely on kunit_test_suite().
1214 static int __init kcsan_test_init(void)
1217 * Because we want to be able to build the test as a module, we need to
1218 * iterate through all known tracepoints, since the static registration
1221 for_each_kernel_tracepoint(register_tracepoints, NULL);
1222 return __kunit_test_suites_init(kcsan_test_suites);
1225 static void kcsan_test_exit(void)
1227 __kunit_test_suites_exit(kcsan_test_suites);
1228 for_each_kernel_tracepoint(unregister_tracepoints, NULL);
1229 tracepoint_synchronize_unregister();
1232 late_initcall(kcsan_test_init);
1233 module_exit(kcsan_test_exit);
1235 MODULE_LICENSE("GPL v2");
1236 MODULE_AUTHOR("Marco Elver <elver@google.com>");