kernel/kcsan/kcsan_test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * KCSAN test with various race scenarious to test runtime behaviour. Since the
   4  * interface with which KCSAN's reports are obtained is via the console, this is
   5  * the output we should verify. For each test case checks the presence (or
   6  * absence) of generated reports. Relies on 'console' tracepoint to capture
   7  * reports as they appear in the kernel log.
   8  *
   9  * Makes use of KUnit for test organization, and the Torture framework for test
  10  * thread control.
  11  *
  12  * Copyright (C) 2020, Google LLC.
  13  * Author: Marco Elver <elver@google.com>
  14  */
  15
  16 #define pr_fmt(fmt) "kcsan_test: " fmt
  17
  18 #include <kunit/test.h>
  19 #include <linux/jiffies.h>
  20 #include <linux/kcsan-checks.h>
  21 #include <linux/kernel.h>
  22 #include <linux/sched.h>
  23 #include <linux/seqlock.h>
  24 #include <linux/spinlock.h>
  25 #include <linux/string.h>
  26 #include <linux/timer.h>
  27 #include <linux/torture.h>
  28 #include <linux/tracepoint.h>
  29 #include <linux/types.h>
  30 #include <trace/events/printk.h>
  31
  32 #ifdef CONFIG_CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE
  33 #define __KCSAN_ACCESS_RW(alt) (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
  34 #else
  35 #define __KCSAN_ACCESS_RW(alt) (alt)
  36 #endif
  37
  38 /* Points to current test-case memory access "kernels". */
  39 static void (*access_kernels[2])(void);
  40
  41 static struct task_struct **threads; /* Lists of threads. */
  42 static unsigned long end_time;       /* End time of test. */
  43
  44 /* Report as observed from console. */
  45 static struct {
  46         spinlock_t lock;
  47         int nlines;
  48         char lines[3][512];
  49 } observed = {
  50         .lock = __SPIN_LOCK_UNLOCKED(observed.lock),
  51 };
  52
  53 /* Setup test checking loop. */
  54 static __no_kcsan inline void
  55 begin_test_checks(void (*func1)(void), void (*func2)(void))
  56 {
  57         kcsan_disable_current();
  58
  59         /*
  60          * Require at least as long as KCSAN_REPORT_ONCE_IN_MS, to ensure at
  61          * least one race is reported.
  62          */
  63         end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500);
  64
  65         /* Signal start; release potential initialization of shared data. */
  66         smp_store_release(&access_kernels[0], func1);
  67         smp_store_release(&access_kernels[1], func2);
  68 }
  69
  70 /* End test checking loop. */
  71 static __no_kcsan inline bool
  72 end_test_checks(bool stop)
  73 {
  74         if (!stop && time_before(jiffies, end_time)) {
  75                 /* Continue checking */
  76                 might_sleep();
  77                 return false;
  78         }
  79
  80         kcsan_enable_current();
  81         return true;
  82 }
  83
  84 /*
  85  * Probe for console output: checks if a race was reported, and obtains observed
  86  * lines of interest.
  87  */
  88 __no_kcsan
  89 static void probe_console(void *ignore, const char *buf, size_t len)
  90 {
  91         unsigned long flags;
  92         int nlines;
  93
  94         /*
  95          * Note that KCSAN reports under a global lock, so we do not risk the
  96          * possibility of having multiple reports interleaved. If that were the
  97          * case, we'd expect tests to fail.
  98          */
  99
 100         spin_lock_irqsave(&observed.lock, flags);
 101         nlines = observed.nlines;
 102
 103         if (strnstr(buf, "BUG: KCSAN: ", len) && strnstr(buf, "test_", len)) {
 104                 /*
 105                  * KCSAN report and related to the test.
 106                  *
 107                  * The provided @buf is not NUL-terminated; copy no more than
 108                  * @len bytes and let strscpy() add the missing NUL-terminator.
 109                  */
 110                 strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0])));
 111                 nlines = 1;
 112         } else if ((nlines == 1 || nlines == 2) && strnstr(buf, "bytes by", len)) {
 113                 strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0])));
 114
 115                 if (strnstr(buf, "race at unknown origin", len)) {
 116                         if (WARN_ON(nlines != 2))
 117                                 goto out;
 118
 119                         /* No second line of interest. */
 120                         strcpy(observed.lines[nlines++], "<none>");
 121                 }
 122         }
 123
 124 out:
 125         WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */
 126         spin_unlock_irqrestore(&observed.lock, flags);
 127 }
 128
 129 /* Check if a report related to the test exists. */
 130 __no_kcsan
 131 static bool report_available(void)
 132 {
 133         return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines);
 134 }
 135
 136 /* Report information we expect in a report. */
 137 struct expect_report {
 138         /* Access information of both accesses. */
 139         struct {
 140                 void *fn;    /* Function pointer to expected function of top frame. */
 141                 void *addr;  /* Address of access; unchecked if NULL. */
 142                 size_t size; /* Size of access; unchecked if @addr is NULL. */
 143                 int type;    /* Access type, see KCSAN_ACCESS definitions. */
 144         } access[2];
 145 };
 146
 147 /* Check observed report matches information in @r. */
 148 __no_kcsan
 149 static bool report_matches(const struct expect_report *r)
 150 {
 151         const bool is_assert = (r->access[0].type | r->access[1].type) & KCSAN_ACCESS_ASSERT;
 152         bool ret = false;
 153         unsigned long flags;
 154         typeof(*observed.lines) *expect;
 155         const char *end;
 156         char *cur;
 157         int i;
 158
 159         /* Doubled-checked locking. */
 160         if (!report_available())
 161                 return false;
 162
 163         expect = kmalloc(sizeof(observed.lines), GFP_KERNEL);
 164         if (WARN_ON(!expect))
 165                 return false;
 166
 167         /* Generate expected report contents. */
 168
 169         /* Title */
 170         cur = expect[0];
 171         end = &expect[0][sizeof(expect[0]) - 1];
 172         cur += scnprintf(cur, end - cur, "BUG: KCSAN: %s in ",
 173                          is_assert ? "assert: race" : "data-race");
 174         if (r->access[1].fn) {
 175                 char tmp[2][64];
 176                 int cmp;
 177
 178                 /* Expect lexographically sorted function names in title. */
 179                 scnprintf(tmp[0], sizeof(tmp[0]), "%pS", r->access[0].fn);
 180                 scnprintf(tmp[1], sizeof(tmp[1]), "%pS", r->access[1].fn);
 181                 cmp = strcmp(tmp[0], tmp[1]);
 182                 cur += scnprintf(cur, end - cur, "%ps / %ps",
 183                                  cmp < 0 ? r->access[0].fn : r->access[1].fn,
 184                                  cmp < 0 ? r->access[1].fn : r->access[0].fn);
 185         } else {
 186                 scnprintf(cur, end - cur, "%pS", r->access[0].fn);
 187                 /* The exact offset won't match, remove it. */
 188                 cur = strchr(expect[0], '+');
 189                 if (cur)
 190                         *cur = '\0';
 191         }
 192
 193         /* Access 1 */
 194         cur = expect[1];
 195         end = &expect[1][sizeof(expect[1]) - 1];
 196         if (!r->access[1].fn)
 197                 cur += scnprintf(cur, end - cur, "race at unknown origin, with ");
 198
 199         /* Access 1 & 2 */
 200         for (i = 0; i < 2; ++i) {
 201                 const int ty = r->access[i].type;
 202                 const char *const access_type =
 203                         (ty & KCSAN_ACCESS_ASSERT) ?
 204                                       ((ty & KCSAN_ACCESS_WRITE) ?
 205                                                "assert no accesses" :
 206                                                "assert no writes") :
 207                                       ((ty & KCSAN_ACCESS_WRITE) ?
 208                                                ((ty & KCSAN_ACCESS_COMPOUND) ?
 209                                                         "read-write" :
 210                                                         "write") :
 211                                                "read");
 212                 const char *const access_type_aux =
 213                         (ty & KCSAN_ACCESS_ATOMIC) ?
 214                                       " (marked)" :
 215                                       ((ty & KCSAN_ACCESS_SCOPED) ? " (scoped)" : "");
 216
 217                 if (i == 1) {
 218                         /* Access 2 */
 219                         cur = expect[2];
 220                         end = &expect[2][sizeof(expect[2]) - 1];
 221
 222                         if (!r->access[1].fn) {
 223                                 /* Dummy string if no second access is available. */
 224                                 strcpy(cur, "<none>");
 225                                 break;
 226                         }
 227                 }
 228
 229                 cur += scnprintf(cur, end - cur, "%s%s to ", access_type,
 230                                  access_type_aux);
 231
 232                 if (r->access[i].addr) /* Address is optional. */
 233                         cur += scnprintf(cur, end - cur, "0x%px of %zu bytes",
 234                                          r->access[i].addr, r->access[i].size);
 235         }
 236
 237         spin_lock_irqsave(&observed.lock, flags);
 238         if (!report_available())
 239                 goto out; /* A new report is being captured. */
 240
 241         /* Finally match expected output to what we actually observed. */
 242         ret = strstr(observed.lines[0], expect[0]) &&
 243               /* Access info may appear in any order. */
 244               ((strstr(observed.lines[1], expect[1]) &&
 245                 strstr(observed.lines[2], expect[2])) ||
 246                (strstr(observed.lines[1], expect[2]) &&
 247                 strstr(observed.lines[2], expect[1])));
 248 out:
 249         spin_unlock_irqrestore(&observed.lock, flags);
 250         kfree(expect);
 251         return ret;
 252 }
 253
 254 /* ===== Test kernels ===== */
 255
 256 static long test_sink;
 257 static long test_var;
 258 /* @test_array should be large enough to fall into multiple watchpoint slots. */
 259 static long test_array[3 * PAGE_SIZE / sizeof(long)];
 260 static struct {
 261         long val[8];
 262 } test_struct;
 263 static DEFINE_SEQLOCK(test_seqlock);
 264
 265 /*
 266  * Helper to avoid compiler optimizing out reads, and to generate source values
 267  * for writes.
 268  */
 269 __no_kcsan
 270 static noinline void sink_value(long v) { WRITE_ONCE(test_sink, v); }
 271
 272 static noinline void test_kernel_read(void) { sink_value(test_var); }
 273
 274 static noinline void test_kernel_write(void)
 275 {
 276         test_var = READ_ONCE_NOCHECK(test_sink) + 1;
 277 }
 278
 279 static noinline void test_kernel_write_nochange(void) { test_var = 42; }
 280
 281 /* Suffixed by value-change exception filter. */
 282 static noinline void test_kernel_write_nochange_rcu(void) { test_var = 42; }
 283
 284 static noinline void test_kernel_read_atomic(void)
 285 {
 286         sink_value(READ_ONCE(test_var));
 287 }
 288
 289 static noinline void test_kernel_write_atomic(void)
 290 {
 291         WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1);
 292 }
 293
 294 static noinline void test_kernel_atomic_rmw(void)
 295 {
 296         /* Use builtin, so we can set up the "bad" atomic/non-atomic scenario. */
 297         __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED);
 298 }
 299
 300 __no_kcsan
 301 static noinline void test_kernel_write_uninstrumented(void) { test_var++; }
 302
 303 static noinline void test_kernel_data_race(void) { data_race(test_var++); }
 304
 305 static noinline void test_kernel_assert_writer(void)
 306 {
 307         ASSERT_EXCLUSIVE_WRITER(test_var);
 308 }
 309
 310 static noinline void test_kernel_assert_access(void)
 311 {
 312         ASSERT_EXCLUSIVE_ACCESS(test_var);
 313 }
 314
 315 #define TEST_CHANGE_BITS 0xff00ff00
 316
 317 static noinline void test_kernel_change_bits(void)
 318 {
 319         if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {
 320                 /*
 321                  * Avoid race of unknown origin for this test, just pretend they
 322                  * are atomic.
 323                  */
 324                 kcsan_nestable_atomic_begin();
 325                 test_var ^= TEST_CHANGE_BITS;
 326                 kcsan_nestable_atomic_end();
 327         } else
 328                 WRITE_ONCE(test_var, READ_ONCE(test_var) ^ TEST_CHANGE_BITS);
 329 }
 330
 331 static noinline void test_kernel_assert_bits_change(void)
 332 {
 333         ASSERT_EXCLUSIVE_BITS(test_var, TEST_CHANGE_BITS);
 334 }
 335
 336 static noinline void test_kernel_assert_bits_nochange(void)
 337 {
 338         ASSERT_EXCLUSIVE_BITS(test_var, ~TEST_CHANGE_BITS);
 339 }
 340
 341 /* To check that scoped assertions do trigger anywhere in scope. */
 342 static noinline void test_enter_scope(void)
 343 {
 344         int x = 0;
 345
 346         /* Unrelated accesses to scoped assert. */
 347         READ_ONCE(test_sink);
 348         kcsan_check_read(&x, sizeof(x));
 349 }
 350
 351 static noinline void test_kernel_assert_writer_scoped(void)
 352 {
 353         ASSERT_EXCLUSIVE_WRITER_SCOPED(test_var);
 354         test_enter_scope();
 355 }
 356
 357 static noinline void test_kernel_assert_access_scoped(void)
 358 {
 359         ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_var);
 360         test_enter_scope();
 361 }
 362
 363 static noinline void test_kernel_rmw_array(void)
 364 {
 365         int i;
 366
 367         for (i = 0; i < ARRAY_SIZE(test_array); ++i)
 368                 test_array[i]++;
 369 }
 370
 371 static noinline void test_kernel_write_struct(void)
 372 {
 373         kcsan_check_write(&test_struct, sizeof(test_struct));
 374         kcsan_disable_current();
 375         test_struct.val[3]++; /* induce value change */
 376         kcsan_enable_current();
 377 }
 378
 379 static noinline void test_kernel_write_struct_part(void)
 380 {
 381         test_struct.val[3] = 42;
 382 }
 383
 384 static noinline void test_kernel_read_struct_zero_size(void)
 385 {
 386         kcsan_check_read(&test_struct.val[3], 0);
 387 }
 388
 389 static noinline void test_kernel_jiffies_reader(void)
 390 {
 391         sink_value((long)jiffies);
 392 }
 393
 394 static noinline void test_kernel_seqlock_reader(void)
 395 {
 396         unsigned int seq;
 397
 398         do {
 399                 seq = read_seqbegin(&test_seqlock);
 400                 sink_value(test_var);
 401         } while (read_seqretry(&test_seqlock, seq));
 402 }
 403
 404 static noinline void test_kernel_seqlock_writer(void)
 405 {
 406         unsigned long flags;
 407
 408         write_seqlock_irqsave(&test_seqlock, flags);
 409         test_var++;
 410         write_sequnlock_irqrestore(&test_seqlock, flags);
 411 }
 412
 413 static noinline void test_kernel_atomic_builtins(void)
 414 {
 415         /*
 416          * Generate concurrent accesses, expecting no reports, ensuring KCSAN
 417          * treats builtin atomics as actually atomic.
 418          */
 419         __atomic_load_n(&test_var, __ATOMIC_RELAXED);
 420 }
 421
 422 static noinline void test_kernel_xor_1bit(void)
 423 {
 424         /* Do not report data races between the read-writes. */
 425         kcsan_nestable_atomic_begin();
 426         test_var ^= 0x10000;
 427         kcsan_nestable_atomic_end();
 428 }
 429
 430 /* ===== Test cases ===== */
 431
 432 /* Simple test with normal data race. */
 433 __no_kcsan
 434 static void test_basic(struct kunit *test)
 435 {
 436         const struct expect_report expect = {
 437                 .access = {
 438                         { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 439                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 440                 },
 441         };
 442         static const struct expect_report never = {
 443                 .access = {
 444                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 445                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 446                 },
 447         };
 448         bool match_expect = false;
 449         bool match_never = false;
 450
 451         begin_test_checks(test_kernel_write, test_kernel_read);
 452         do {
 453                 match_expect |= report_matches(&expect);
 454                 match_never = report_matches(&never);
 455         } while (!end_test_checks(match_never));
 456         KUNIT_EXPECT_TRUE(test, match_expect);
 457         KUNIT_EXPECT_FALSE(test, match_never);
 458 }
 459
 460 /*
 461  * Stress KCSAN with lots of concurrent races on different addresses until
 462  * timeout.
 463  */
 464 __no_kcsan
 465 static void test_concurrent_races(struct kunit *test)
 466 {
 467         const struct expect_report expect = {
 468                 .access = {
 469                         /* NULL will match any address. */
 470                         { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
 471                         { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(0) },
 472                 },
 473         };
 474         static const struct expect_report never = {
 475                 .access = {
 476                         { test_kernel_rmw_array, NULL, 0, 0 },
 477                         { test_kernel_rmw_array, NULL, 0, 0 },
 478                 },
 479         };
 480         bool match_expect = false;
 481         bool match_never = false;
 482
 483         begin_test_checks(test_kernel_rmw_array, test_kernel_rmw_array);
 484         do {
 485                 match_expect |= report_matches(&expect);
 486                 match_never |= report_matches(&never);
 487         } while (!end_test_checks(false));
 488         KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check matches exist. */
 489         KUNIT_EXPECT_FALSE(test, match_never);
 490 }
 491
 492 /* Test the KCSAN_REPORT_VALUE_CHANGE_ONLY option. */
 493 __no_kcsan
 494 static void test_novalue_change(struct kunit *test)
 495 {
 496         const struct expect_report expect = {
 497                 .access = {
 498                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 499                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 500                 },
 501         };
 502         bool match_expect = false;
 503
 504         begin_test_checks(test_kernel_write_nochange, test_kernel_read);
 505         do {
 506                 match_expect = report_matches(&expect);
 507         } while (!end_test_checks(match_expect));
 508         if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY))
 509                 KUNIT_EXPECT_FALSE(test, match_expect);
 510         else
 511                 KUNIT_EXPECT_TRUE(test, match_expect);
 512 }
 513
 514 /*
 515  * Test that the rules where the KCSAN_REPORT_VALUE_CHANGE_ONLY option should
 516  * never apply work.
 517  */
 518 __no_kcsan
 519 static void test_novalue_change_exception(struct kunit *test)
 520 {
 521         const struct expect_report expect = {
 522                 .access = {
 523                         { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 524                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 525                 },
 526         };
 527         bool match_expect = false;
 528
 529         begin_test_checks(test_kernel_write_nochange_rcu, test_kernel_read);
 530         do {
 531                 match_expect = report_matches(&expect);
 532         } while (!end_test_checks(match_expect));
 533         KUNIT_EXPECT_TRUE(test, match_expect);
 534 }
 535
 536 /* Test that data races of unknown origin are reported. */
 537 __no_kcsan
 538 static void test_unknown_origin(struct kunit *test)
 539 {
 540         const struct expect_report expect = {
 541                 .access = {
 542                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 543                         { NULL },
 544                 },
 545         };
 546         bool match_expect = false;
 547
 548         begin_test_checks(test_kernel_write_uninstrumented, test_kernel_read);
 549         do {
 550                 match_expect = report_matches(&expect);
 551         } while (!end_test_checks(match_expect));
 552         if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN))
 553                 KUNIT_EXPECT_TRUE(test, match_expect);
 554         else
 555                 KUNIT_EXPECT_FALSE(test, match_expect);
 556 }
 557
 558 /* Test KCSAN_ASSUME_PLAIN_WRITES_ATOMIC if it is selected. */
 559 __no_kcsan
 560 static void test_write_write_assume_atomic(struct kunit *test)
 561 {
 562         const struct expect_report expect = {
 563                 .access = {
 564                         { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 565                         { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 566                 },
 567         };
 568         bool match_expect = false;
 569
 570         begin_test_checks(test_kernel_write, test_kernel_write);
 571         do {
 572                 sink_value(READ_ONCE(test_var)); /* induce value-change */
 573                 match_expect = report_matches(&expect);
 574         } while (!end_test_checks(match_expect));
 575         if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC))
 576                 KUNIT_EXPECT_FALSE(test, match_expect);
 577         else
 578                 KUNIT_EXPECT_TRUE(test, match_expect);
 579 }
 580
 581 /*
 582  * Test that data races with writes larger than word-size are always reported,
 583  * even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
 584  */
 585 __no_kcsan
 586 static void test_write_write_struct(struct kunit *test)
 587 {
 588         const struct expect_report expect = {
 589                 .access = {
 590                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 591                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 592                 },
 593         };
 594         bool match_expect = false;
 595
 596         begin_test_checks(test_kernel_write_struct, test_kernel_write_struct);
 597         do {
 598                 match_expect = report_matches(&expect);
 599         } while (!end_test_checks(match_expect));
 600         KUNIT_EXPECT_TRUE(test, match_expect);
 601 }
 602
 603 /*
 604  * Test that data races where only one write is larger than word-size are always
 605  * reported, even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
 606  */
 607 __no_kcsan
 608 static void test_write_write_struct_part(struct kunit *test)
 609 {
 610         const struct expect_report expect = {
 611                 .access = {
 612                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 613                         { test_kernel_write_struct_part, &test_struct.val[3], sizeof(test_struct.val[3]), KCSAN_ACCESS_WRITE },
 614                 },
 615         };
 616         bool match_expect = false;
 617
 618         begin_test_checks(test_kernel_write_struct, test_kernel_write_struct_part);
 619         do {
 620                 match_expect = report_matches(&expect);
 621         } while (!end_test_checks(match_expect));
 622         KUNIT_EXPECT_TRUE(test, match_expect);
 623 }
 624
 625 /* Test that races with atomic accesses never result in reports. */
 626 __no_kcsan
 627 static void test_read_atomic_write_atomic(struct kunit *test)
 628 {
 629         bool match_never = false;
 630
 631         begin_test_checks(test_kernel_read_atomic, test_kernel_write_atomic);
 632         do {
 633                 match_never = report_available();
 634         } while (!end_test_checks(match_never));
 635         KUNIT_EXPECT_FALSE(test, match_never);
 636 }
 637
 638 /* Test that a race with an atomic and plain access result in reports. */
 639 __no_kcsan
 640 static void test_read_plain_atomic_write(struct kunit *test)
 641 {
 642         const struct expect_report expect = {
 643                 .access = {
 644                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 645                         { test_kernel_write_atomic, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
 646                 },
 647         };
 648         bool match_expect = false;
 649
 650         if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS))
 651                 return;
 652
 653         begin_test_checks(test_kernel_read, test_kernel_write_atomic);
 654         do {
 655                 match_expect = report_matches(&expect);
 656         } while (!end_test_checks(match_expect));
 657         KUNIT_EXPECT_TRUE(test, match_expect);
 658 }
 659
 660 /* Test that atomic RMWs generate correct report. */
 661 __no_kcsan
 662 static void test_read_plain_atomic_rmw(struct kunit *test)
 663 {
 664         const struct expect_report expect = {
 665                 .access = {
 666                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 667                         { test_kernel_atomic_rmw, &test_var, sizeof(test_var),
 668                                 KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
 669                 },
 670         };
 671         bool match_expect = false;
 672
 673         if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS))
 674                 return;
 675
 676         begin_test_checks(test_kernel_read, test_kernel_atomic_rmw);
 677         do {
 678                 match_expect = report_matches(&expect);
 679         } while (!end_test_checks(match_expect));
 680         KUNIT_EXPECT_TRUE(test, match_expect);
 681 }
 682
 683 /* Zero-sized accesses should never cause data race reports. */
 684 __no_kcsan
 685 static void test_zero_size_access(struct kunit *test)
 686 {
 687         const struct expect_report expect = {
 688                 .access = {
 689                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 690                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 691                 },
 692         };
 693         const struct expect_report never = {
 694                 .access = {
 695                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 696                         { test_kernel_read_struct_zero_size, &test_struct.val[3], 0, 0 },
 697                 },
 698         };
 699         bool match_expect = false;
 700         bool match_never = false;
 701
 702         begin_test_checks(test_kernel_write_struct, test_kernel_read_struct_zero_size);
 703         do {
 704                 match_expect |= report_matches(&expect);
 705                 match_never = report_matches(&never);
 706         } while (!end_test_checks(match_never));
 707         KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check. */
 708         KUNIT_EXPECT_FALSE(test, match_never);
 709 }
 710
 711 /* Test the data_race() macro. */
 712 __no_kcsan
 713 static void test_data_race(struct kunit *test)
 714 {
 715         bool match_never = false;
 716
 717         begin_test_checks(test_kernel_data_race, test_kernel_data_race);
 718         do {
 719                 match_never = report_available();
 720         } while (!end_test_checks(match_never));
 721         KUNIT_EXPECT_FALSE(test, match_never);
 722 }
 723
 724 __no_kcsan
 725 static void test_assert_exclusive_writer(struct kunit *test)
 726 {
 727         const struct expect_report expect = {
 728                 .access = {
 729                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
 730                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 731                 },
 732         };
 733         bool match_expect = false;
 734
 735         begin_test_checks(test_kernel_assert_writer, test_kernel_write_nochange);
 736         do {
 737                 match_expect = report_matches(&expect);
 738         } while (!end_test_checks(match_expect));
 739         KUNIT_EXPECT_TRUE(test, match_expect);
 740 }
 741
 742 __no_kcsan
 743 static void test_assert_exclusive_access(struct kunit *test)
 744 {
 745         const struct expect_report expect = {
 746                 .access = {
 747                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
 748                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 749                 },
 750         };
 751         bool match_expect = false;
 752
 753         begin_test_checks(test_kernel_assert_access, test_kernel_read);
 754         do {
 755                 match_expect = report_matches(&expect);
 756         } while (!end_test_checks(match_expect));
 757         KUNIT_EXPECT_TRUE(test, match_expect);
 758 }
 759
 760 __no_kcsan
 761 static void test_assert_exclusive_access_writer(struct kunit *test)
 762 {
 763         const struct expect_report expect_access_writer = {
 764                 .access = {
 765                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
 766                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
 767                 },
 768         };
 769         const struct expect_report expect_access_access = {
 770                 .access = {
 771                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
 772                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
 773                 },
 774         };
 775         const struct expect_report never = {
 776                 .access = {
 777                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
 778                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
 779                 },
 780         };
 781         bool match_expect_access_writer = false;
 782         bool match_expect_access_access = false;
 783         bool match_never = false;
 784
 785         begin_test_checks(test_kernel_assert_access, test_kernel_assert_writer);
 786         do {
 787                 match_expect_access_writer |= report_matches(&expect_access_writer);
 788                 match_expect_access_access |= report_matches(&expect_access_access);
 789                 match_never |= report_matches(&never);
 790         } while (!end_test_checks(match_never));
 791         KUNIT_EXPECT_TRUE(test, match_expect_access_writer);
 792         KUNIT_EXPECT_TRUE(test, match_expect_access_access);
 793         KUNIT_EXPECT_FALSE(test, match_never);
 794 }
 795
 796 __no_kcsan
 797 static void test_assert_exclusive_bits_change(struct kunit *test)
 798 {
 799         const struct expect_report expect = {
 800                 .access = {
 801                         { test_kernel_assert_bits_change, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
 802                         { test_kernel_change_bits, &test_var, sizeof(test_var),
 803                                 KCSAN_ACCESS_WRITE | (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) ? 0 : KCSAN_ACCESS_ATOMIC) },
 804                 },
 805         };
 806         bool match_expect = false;
 807
 808         begin_test_checks(test_kernel_assert_bits_change, test_kernel_change_bits);
 809         do {
 810                 match_expect = report_matches(&expect);
 811         } while (!end_test_checks(match_expect));
 812         KUNIT_EXPECT_TRUE(test, match_expect);
 813 }
 814
 815 __no_kcsan
 816 static void test_assert_exclusive_bits_nochange(struct kunit *test)
 817 {
 818         bool match_never = false;
 819
 820         begin_test_checks(test_kernel_assert_bits_nochange, test_kernel_change_bits);
 821         do {
 822                 match_never = report_available();
 823         } while (!end_test_checks(match_never));
 824         KUNIT_EXPECT_FALSE(test, match_never);
 825 }
 826
 827 __no_kcsan
 828 static void test_assert_exclusive_writer_scoped(struct kunit *test)
 829 {
 830         const struct expect_report expect_start = {
 831                 .access = {
 832                         { test_kernel_assert_writer_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
 833                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 834                 },
 835         };
 836         const struct expect_report expect_anywhere = {
 837                 .access = {
 838                         { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
 839                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 840                 },
 841         };
 842         bool match_expect_start = false;
 843         bool match_expect_anywhere = false;
 844
 845         begin_test_checks(test_kernel_assert_writer_scoped, test_kernel_write_nochange);
 846         do {
 847                 match_expect_start |= report_matches(&expect_start);
 848                 match_expect_anywhere |= report_matches(&expect_anywhere);
 849         } while (!end_test_checks(match_expect_start && match_expect_anywhere));
 850         KUNIT_EXPECT_TRUE(test, match_expect_start);
 851         KUNIT_EXPECT_TRUE(test, match_expect_anywhere);
 852 }
 853
 854 __no_kcsan
 855 static void test_assert_exclusive_access_scoped(struct kunit *test)
 856 {
 857         const struct expect_report expect_start1 = {
 858                 .access = {
 859                         { test_kernel_assert_access_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
 860                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 861                 },
 862         };
 863         const struct expect_report expect_start2 = {
 864                 .access = { expect_start1.access[0], expect_start1.access[0] },
 865         };
 866         const struct expect_report expect_inscope = {
 867                 .access = {
 868                         { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
 869                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 870                 },
 871         };
 872         bool match_expect_start = false;
 873         bool match_expect_inscope = false;
 874
 875         begin_test_checks(test_kernel_assert_access_scoped, test_kernel_read);
 876         end_time += msecs_to_jiffies(1000); /* This test requires a bit more time. */
 877         do {
 878                 match_expect_start |= report_matches(&expect_start1) || report_matches(&expect_start2);
 879                 match_expect_inscope |= report_matches(&expect_inscope);
 880         } while (!end_test_checks(match_expect_start && match_expect_inscope));
 881         KUNIT_EXPECT_TRUE(test, match_expect_start);
 882         KUNIT_EXPECT_TRUE(test, match_expect_inscope);
 883 }
 884
 885 /*
 886  * jiffies is special (declared to be volatile) and its accesses are typically
 887  * not marked; this test ensures that the compiler nor KCSAN gets confused about
 888  * jiffies's declaration on different architectures.
 889  */
 890 __no_kcsan
 891 static void test_jiffies_noreport(struct kunit *test)
 892 {
 893         bool match_never = false;
 894
 895         begin_test_checks(test_kernel_jiffies_reader, test_kernel_jiffies_reader);
 896         do {
 897                 match_never = report_available();
 898         } while (!end_test_checks(match_never));
 899         KUNIT_EXPECT_FALSE(test, match_never);
 900 }
 901
 902 /* Test that racing accesses in seqlock critical sections are not reported. */
 903 __no_kcsan
 904 static void test_seqlock_noreport(struct kunit *test)
 905 {
 906         bool match_never = false;
 907
 908         begin_test_checks(test_kernel_seqlock_reader, test_kernel_seqlock_writer);
 909         do {
 910                 match_never = report_available();
 911         } while (!end_test_checks(match_never));
 912         KUNIT_EXPECT_FALSE(test, match_never);
 913 }
 914
 915 /*
 916  * Test atomic builtins work and required instrumentation functions exist. We
 917  * also test that KCSAN understands they're atomic by racing with them via
 918  * test_kernel_atomic_builtins(), and expect no reports.
 919  *
 920  * The atomic builtins _SHOULD NOT_ be used in normal kernel code!
 921  */
 922 static void test_atomic_builtins(struct kunit *test)
 923 {
 924         bool match_never = false;
 925
 926         begin_test_checks(test_kernel_atomic_builtins, test_kernel_atomic_builtins);
 927         do {
 928                 long tmp;
 929
 930                 kcsan_enable_current();
 931
 932                 __atomic_store_n(&test_var, 42L, __ATOMIC_RELAXED);
 933                 KUNIT_EXPECT_EQ(test, 42L, __atomic_load_n(&test_var, __ATOMIC_RELAXED));
 934
 935                 KUNIT_EXPECT_EQ(test, 42L, __atomic_exchange_n(&test_var, 20, __ATOMIC_RELAXED));
 936                 KUNIT_EXPECT_EQ(test, 20L, test_var);
 937
 938                 tmp = 20L;
 939                 KUNIT_EXPECT_TRUE(test, __atomic_compare_exchange_n(&test_var, &tmp, 30L,
 940                                                                     0, __ATOMIC_RELAXED,
 941                                                                     __ATOMIC_RELAXED));
 942                 KUNIT_EXPECT_EQ(test, tmp, 20L);
 943                 KUNIT_EXPECT_EQ(test, test_var, 30L);
 944                 KUNIT_EXPECT_FALSE(test, __atomic_compare_exchange_n(&test_var, &tmp, 40L,
 945                                                                      1, __ATOMIC_RELAXED,
 946                                                                      __ATOMIC_RELAXED));
 947                 KUNIT_EXPECT_EQ(test, tmp, 30L);
 948                 KUNIT_EXPECT_EQ(test, test_var, 30L);
 949
 950                 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED));
 951                 KUNIT_EXPECT_EQ(test, 31L, __atomic_fetch_sub(&test_var, 1, __ATOMIC_RELAXED));
 952                 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_and(&test_var, 0xf, __ATOMIC_RELAXED));
 953                 KUNIT_EXPECT_EQ(test, 14L, __atomic_fetch_xor(&test_var, 0xf, __ATOMIC_RELAXED));
 954                 KUNIT_EXPECT_EQ(test, 1L, __atomic_fetch_or(&test_var, 0xf0, __ATOMIC_RELAXED));
 955                 KUNIT_EXPECT_EQ(test, 241L, __atomic_fetch_nand(&test_var, 0xf, __ATOMIC_RELAXED));
 956                 KUNIT_EXPECT_EQ(test, -2L, test_var);
 957
 958                 __atomic_thread_fence(__ATOMIC_SEQ_CST);
 959                 __atomic_signal_fence(__ATOMIC_SEQ_CST);
 960
 961                 kcsan_disable_current();
 962
 963                 match_never = report_available();
 964         } while (!end_test_checks(match_never));
 965         KUNIT_EXPECT_FALSE(test, match_never);
 966 }
 967
 968 __no_kcsan
 969 static void test_1bit_value_change(struct kunit *test)
 970 {
 971         const struct expect_report expect = {
 972                 .access = {
 973                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 974                         { test_kernel_xor_1bit, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
 975                 },
 976         };
 977         bool match = false;
 978
 979         begin_test_checks(test_kernel_read, test_kernel_xor_1bit);
 980         do {
 981                 match = IS_ENABLED(CONFIG_KCSAN_PERMISSIVE)
 982                                 ? report_available()
 983                                 : report_matches(&expect);
 984         } while (!end_test_checks(match));
 985         if (IS_ENABLED(CONFIG_KCSAN_PERMISSIVE))
 986                 KUNIT_EXPECT_FALSE(test, match);
 987         else
 988                 KUNIT_EXPECT_TRUE(test, match);
 989 }
 990
 991 /*
 992  * Generate thread counts for all test cases. Values generated are in interval
 993  * [2, 5] followed by exponentially increasing thread counts from 8 to 32.
 994  *
 995  * The thread counts are chosen to cover potentially interesting boundaries and
 996  * corner cases (2 to 5), and then stress the system with larger counts.
 997  */
 998 static const void *nthreads_gen_params(const void *prev, char *desc)
 999 {
1000         long nthreads = (long)prev;
1001
1002         if (nthreads < 0 || nthreads >= 32)
1003                 nthreads = 0; /* stop */
1004         else if (!nthreads)
1005                 nthreads = 2; /* initial value */
1006         else if (nthreads < 5)
1007                 nthreads++;
1008         else if (nthreads == 5)
1009                 nthreads = 8;
1010         else
1011                 nthreads *= 2;
1012
1013         if (!IS_ENABLED(CONFIG_PREEMPT) || !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
1014                 /*
1015                  * Without any preemption, keep 2 CPUs free for other tasks, one
1016                  * of which is the main test case function checking for
1017                  * completion or failure.
1018                  */
1019                 const long min_unused_cpus = IS_ENABLED(CONFIG_PREEMPT_NONE) ? 2 : 0;
1020                 const long min_required_cpus = 2 + min_unused_cpus;
1021
1022                 if (num_online_cpus() < min_required_cpus) {
1023                         pr_err_once("Too few online CPUs (%u < %ld) for test\n",
1024                                     num_online_cpus(), min_required_cpus);
1025                         nthreads = 0;
1026                 } else if (nthreads >= num_online_cpus() - min_unused_cpus) {
1027                         /* Use negative value to indicate last param. */
1028                         nthreads = -(num_online_cpus() - min_unused_cpus);
1029                         pr_warn_once("Limiting number of threads to %ld (only %d online CPUs)\n",
1030                                      -nthreads, num_online_cpus());
1031                 }
1032         }
1033
1034         snprintf(desc, KUNIT_PARAM_DESC_SIZE, "threads=%ld", abs(nthreads));
1035         return (void *)nthreads;
1036 }
1037
1038 #define KCSAN_KUNIT_CASE(test_name) KUNIT_CASE_PARAM(test_name, nthreads_gen_params)
1039 static struct kunit_case kcsan_test_cases[] = {
1040         KCSAN_KUNIT_CASE(test_basic),
1041         KCSAN_KUNIT_CASE(test_concurrent_races),
1042         KCSAN_KUNIT_CASE(test_novalue_change),
1043         KCSAN_KUNIT_CASE(test_novalue_change_exception),
1044         KCSAN_KUNIT_CASE(test_unknown_origin),
1045         KCSAN_KUNIT_CASE(test_write_write_assume_atomic),
1046         KCSAN_KUNIT_CASE(test_write_write_struct),
1047         KCSAN_KUNIT_CASE(test_write_write_struct_part),
1048         KCSAN_KUNIT_CASE(test_read_atomic_write_atomic),
1049         KCSAN_KUNIT_CASE(test_read_plain_atomic_write),
1050         KCSAN_KUNIT_CASE(test_read_plain_atomic_rmw),
1051         KCSAN_KUNIT_CASE(test_zero_size_access),
1052         KCSAN_KUNIT_CASE(test_data_race),
1053         KCSAN_KUNIT_CASE(test_assert_exclusive_writer),
1054         KCSAN_KUNIT_CASE(test_assert_exclusive_access),
1055         KCSAN_KUNIT_CASE(test_assert_exclusive_access_writer),
1056         KCSAN_KUNIT_CASE(test_assert_exclusive_bits_change),
1057         KCSAN_KUNIT_CASE(test_assert_exclusive_bits_nochange),
1058         KCSAN_KUNIT_CASE(test_assert_exclusive_writer_scoped),
1059         KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped),
1060         KCSAN_KUNIT_CASE(test_jiffies_noreport),
1061         KCSAN_KUNIT_CASE(test_seqlock_noreport),
1062         KCSAN_KUNIT_CASE(test_atomic_builtins),
1063         KCSAN_KUNIT_CASE(test_1bit_value_change),
1064         {},
1065 };
1066
1067 /* ===== End test cases ===== */
1068
1069 /* Concurrent accesses from interrupts. */
1070 __no_kcsan
1071 static void access_thread_timer(struct timer_list *timer)
1072 {
1073         static atomic_t cnt = ATOMIC_INIT(0);
1074         unsigned int idx;
1075         void (*func)(void);
1076
1077         idx = (unsigned int)atomic_inc_return(&cnt) % ARRAY_SIZE(access_kernels);
1078         /* Acquire potential initialization. */
1079         func = smp_load_acquire(&access_kernels[idx]);
1080         if (func)
1081                 func();
1082 }
1083
1084 /* The main loop for each thread. */
1085 __no_kcsan
1086 static int access_thread(void *arg)
1087 {
1088         struct timer_list timer;
1089         unsigned int cnt = 0;
1090         unsigned int idx;
1091         void (*func)(void);
1092
1093         timer_setup_on_stack(&timer, access_thread_timer, 0);
1094         do {
1095                 might_sleep();
1096
1097                 if (!timer_pending(&timer))
1098                         mod_timer(&timer, jiffies + 1);
1099                 else {
1100                         /* Iterate through all kernels. */
1101                         idx = cnt++ % ARRAY_SIZE(access_kernels);
1102                         /* Acquire potential initialization. */
1103                         func = smp_load_acquire(&access_kernels[idx]);
1104                         if (func)
1105                                 func();
1106                 }
1107         } while (!torture_must_stop());
1108         del_timer_sync(&timer);
1109         destroy_timer_on_stack(&timer);
1110
1111         torture_kthread_stopping("access_thread");
1112         return 0;
1113 }
1114
1115 __no_kcsan
1116 static int test_init(struct kunit *test)
1117 {
1118         unsigned long flags;
1119         int nthreads;
1120         int i;
1121
1122         spin_lock_irqsave(&observed.lock, flags);
1123         for (i = 0; i < ARRAY_SIZE(observed.lines); ++i)
1124                 observed.lines[i][0] = '\0';
1125         observed.nlines = 0;
1126         spin_unlock_irqrestore(&observed.lock, flags);
1127
1128         if (!torture_init_begin((char *)test->name, 1))
1129                 return -EBUSY;
1130
1131         if (WARN_ON(threads))
1132                 goto err;
1133
1134         for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) {
1135                 if (WARN_ON(access_kernels[i]))
1136                         goto err;
1137         }
1138
1139         nthreads = abs((long)test->param_value);
1140         if (WARN_ON(!nthreads))
1141                 goto err;
1142
1143         threads = kcalloc(nthreads + 1, sizeof(struct task_struct *), GFP_KERNEL);
1144         if (WARN_ON(!threads))
1145                 goto err;
1146
1147         threads[nthreads] = NULL;
1148         for (i = 0; i < nthreads; ++i) {
1149                 if (torture_create_kthread(access_thread, NULL, threads[i]))
1150                         goto err;
1151         }
1152
1153         torture_init_end();
1154
1155         return 0;
1156
1157 err:
1158         kfree(threads);
1159         threads = NULL;
1160         torture_init_end();
1161         return -EINVAL;
1162 }
1163
1164 __no_kcsan
1165 static void test_exit(struct kunit *test)
1166 {
1167         struct task_struct **stop_thread;
1168         int i;
1169
1170         if (torture_cleanup_begin())
1171                 return;
1172
1173         for (i = 0; i < ARRAY_SIZE(access_kernels); ++i)
1174                 WRITE_ONCE(access_kernels[i], NULL);
1175
1176         if (threads) {
1177                 for (stop_thread = threads; *stop_thread; stop_thread++)
1178                         torture_stop_kthread(reader_thread, *stop_thread);
1179
1180                 kfree(threads);
1181                 threads = NULL;
1182         }
1183
1184         torture_cleanup_end();
1185 }
1186
1187 static struct kunit_suite kcsan_test_suite = {
1188         .name = "kcsan",
1189         .test_cases = kcsan_test_cases,
1190         .init = test_init,
1191         .exit = test_exit,
1192 };
1193 static struct kunit_suite *kcsan_test_suites[] = { &kcsan_test_suite, NULL };
1194
1195 __no_kcsan
1196 static void register_tracepoints(struct tracepoint *tp, void *ignore)
1197 {
1198         check_trace_callback_type_console(probe_console);
1199         if (!strcmp(tp->name, "console"))
1200                 WARN_ON(tracepoint_probe_register(tp, probe_console, NULL));
1201 }
1202
1203 __no_kcsan
1204 static void unregister_tracepoints(struct tracepoint *tp, void *ignore)
1205 {
1206         if (!strcmp(tp->name, "console"))
1207                 tracepoint_probe_unregister(tp, probe_console, NULL);
1208 }
1209
1210 /*
1211  * We only want to do tracepoints setup and teardown once, therefore we have to
1212  * customize the init and exit functions and cannot rely on kunit_test_suite().
1213  */
1214 static int __init kcsan_test_init(void)
1215 {
1216         /*
1217          * Because we want to be able to build the test as a module, we need to
1218          * iterate through all known tracepoints, since the static registration
1219          * won't work here.
1220          */
1221         for_each_kernel_tracepoint(register_tracepoints, NULL);
1222         return __kunit_test_suites_init(kcsan_test_suites);
1223 }
1224
1225 static void kcsan_test_exit(void)
1226 {
1227         __kunit_test_suites_exit(kcsan_test_suites);
1228         for_each_kernel_tracepoint(unregister_tracepoints, NULL);
1229         tracepoint_synchronize_unregister();
1230 }
1231
1232 late_initcall(kcsan_test_init);
1233 module_exit(kcsan_test_exit);
1234
1235 MODULE_LICENSE("GPL v2");
1236 MODULE_AUTHOR("Marco Elver <elver@google.com>");