lib/locking-selftest.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * lib/locking-selftest.c
   4  *
   5  * Testsuite for various locking APIs: spinlocks, rwlocks,
   6  * mutexes and rw-semaphores.
   7  *
   8  * It is checking both false positives and false negatives.
   9  *
  10  * Started by Ingo Molnar:
  11  *
  12  *  Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  13  */
  14 #include <linux/rwsem.h>
  15 #include <linux/mutex.h>
  16 #include <linux/ww_mutex.h>
  17 #include <linux/sched.h>
  18 #include <linux/sched/mm.h>
  19 #include <linux/delay.h>
  20 #include <linux/lockdep.h>
  21 #include <linux/spinlock.h>
  22 #include <linux/kallsyms.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/debug_locks.h>
  25 #include <linux/irqflags.h>
  26 #include <linux/rtmutex.h>
  27 #include <linux/local_lock.h>
  28
  29 /*
  30  * Change this to 1 if you want to see the failure printouts:
  31  */
  32 static unsigned int debug_locks_verbose;
  33 unsigned int force_read_lock_recursive;
  34
  35 static DEFINE_WD_CLASS(ww_lockdep);
  36
  37 static int __init setup_debug_locks_verbose(char *str)
  38 {
  39         get_option(&str, &debug_locks_verbose);
  40
  41         return 1;
  42 }
  43
  44 __setup("debug_locks_verbose=", setup_debug_locks_verbose);
  45
  46 #define FAILURE         0
  47 #define SUCCESS         1
  48
  49 #define LOCKTYPE_SPIN   0x1
  50 #define LOCKTYPE_RWLOCK 0x2
  51 #define LOCKTYPE_MUTEX  0x4
  52 #define LOCKTYPE_RWSEM  0x8
  53 #define LOCKTYPE_WW     0x10
  54 #define LOCKTYPE_RTMUTEX 0x20
  55 #define LOCKTYPE_LL     0x40
  56
  57 static struct ww_acquire_ctx t, t2;
  58 static struct ww_mutex o, o2, o3;
  59
  60 /*
  61  * Normal standalone locks, for the circular and irq-context
  62  * dependency tests:
  63  */
  64 static DEFINE_SPINLOCK(lock_A);
  65 static DEFINE_SPINLOCK(lock_B);
  66 static DEFINE_SPINLOCK(lock_C);
  67 static DEFINE_SPINLOCK(lock_D);
  68
  69 static DEFINE_RAW_SPINLOCK(raw_lock_A);
  70 static DEFINE_RAW_SPINLOCK(raw_lock_B);
  71
  72 static DEFINE_RWLOCK(rwlock_A);
  73 static DEFINE_RWLOCK(rwlock_B);
  74 static DEFINE_RWLOCK(rwlock_C);
  75 static DEFINE_RWLOCK(rwlock_D);
  76
  77 static DEFINE_MUTEX(mutex_A);
  78 static DEFINE_MUTEX(mutex_B);
  79 static DEFINE_MUTEX(mutex_C);
  80 static DEFINE_MUTEX(mutex_D);
  81
  82 static DECLARE_RWSEM(rwsem_A);
  83 static DECLARE_RWSEM(rwsem_B);
  84 static DECLARE_RWSEM(rwsem_C);
  85 static DECLARE_RWSEM(rwsem_D);
  86
  87 #ifdef CONFIG_RT_MUTEXES
  88
  89 static DEFINE_RT_MUTEX(rtmutex_A);
  90 static DEFINE_RT_MUTEX(rtmutex_B);
  91 static DEFINE_RT_MUTEX(rtmutex_C);
  92 static DEFINE_RT_MUTEX(rtmutex_D);
  93
  94 #endif
  95
  96 /*
  97  * Locks that we initialize dynamically as well so that
  98  * e.g. X1 and X2 becomes two instances of the same class,
  99  * but X* and Y* are different classes. We do this so that
 100  * we do not trigger a real lockup:
 101  */
 102 static DEFINE_SPINLOCK(lock_X1);
 103 static DEFINE_SPINLOCK(lock_X2);
 104 static DEFINE_SPINLOCK(lock_Y1);
 105 static DEFINE_SPINLOCK(lock_Y2);
 106 static DEFINE_SPINLOCK(lock_Z1);
 107 static DEFINE_SPINLOCK(lock_Z2);
 108
 109 static DEFINE_RWLOCK(rwlock_X1);
 110 static DEFINE_RWLOCK(rwlock_X2);
 111 static DEFINE_RWLOCK(rwlock_Y1);
 112 static DEFINE_RWLOCK(rwlock_Y2);
 113 static DEFINE_RWLOCK(rwlock_Z1);
 114 static DEFINE_RWLOCK(rwlock_Z2);
 115
 116 static DEFINE_MUTEX(mutex_X1);
 117 static DEFINE_MUTEX(mutex_X2);
 118 static DEFINE_MUTEX(mutex_Y1);
 119 static DEFINE_MUTEX(mutex_Y2);
 120 static DEFINE_MUTEX(mutex_Z1);
 121 static DEFINE_MUTEX(mutex_Z2);
 122
 123 static DECLARE_RWSEM(rwsem_X1);
 124 static DECLARE_RWSEM(rwsem_X2);
 125 static DECLARE_RWSEM(rwsem_Y1);
 126 static DECLARE_RWSEM(rwsem_Y2);
 127 static DECLARE_RWSEM(rwsem_Z1);
 128 static DECLARE_RWSEM(rwsem_Z2);
 129
 130 #ifdef CONFIG_RT_MUTEXES
 131
 132 static DEFINE_RT_MUTEX(rtmutex_X1);
 133 static DEFINE_RT_MUTEX(rtmutex_X2);
 134 static DEFINE_RT_MUTEX(rtmutex_Y1);
 135 static DEFINE_RT_MUTEX(rtmutex_Y2);
 136 static DEFINE_RT_MUTEX(rtmutex_Z1);
 137 static DEFINE_RT_MUTEX(rtmutex_Z2);
 138
 139 #endif
 140
 141 static local_lock_t local_A = INIT_LOCAL_LOCK(local_A);
 142
 143 /*
 144  * non-inlined runtime initializers, to let separate locks share
 145  * the same lock-class:
 146  */
 147 #define INIT_CLASS_FUNC(class)                          \
 148 static noinline void                                    \
 149 init_class_##class(spinlock_t *lock, rwlock_t *rwlock, \
 150         struct mutex *mutex, struct rw_semaphore *rwsem)\
 151 {                                                       \
 152         spin_lock_init(lock);                   \
 153         rwlock_init(rwlock);                            \
 154         mutex_init(mutex);                              \
 155         init_rwsem(rwsem);                              \
 156 }
 157
 158 INIT_CLASS_FUNC(X)
 159 INIT_CLASS_FUNC(Y)
 160 INIT_CLASS_FUNC(Z)
 161
 162 static void init_shared_classes(void)
 163 {
 164 #ifdef CONFIG_RT_MUTEXES
 165         static struct lock_class_key rt_X, rt_Y, rt_Z;
 166
 167         __rt_mutex_init(&rtmutex_X1, __func__, &rt_X);
 168         __rt_mutex_init(&rtmutex_X2, __func__, &rt_X);
 169         __rt_mutex_init(&rtmutex_Y1, __func__, &rt_Y);
 170         __rt_mutex_init(&rtmutex_Y2, __func__, &rt_Y);
 171         __rt_mutex_init(&rtmutex_Z1, __func__, &rt_Z);
 172         __rt_mutex_init(&rtmutex_Z2, __func__, &rt_Z);
 173 #endif
 174
 175         init_class_X(&lock_X1, &rwlock_X1, &mutex_X1, &rwsem_X1);
 176         init_class_X(&lock_X2, &rwlock_X2, &mutex_X2, &rwsem_X2);
 177
 178         init_class_Y(&lock_Y1, &rwlock_Y1, &mutex_Y1, &rwsem_Y1);
 179         init_class_Y(&lock_Y2, &rwlock_Y2, &mutex_Y2, &rwsem_Y2);
 180
 181         init_class_Z(&lock_Z1, &rwlock_Z1, &mutex_Z1, &rwsem_Z1);
 182         init_class_Z(&lock_Z2, &rwlock_Z2, &mutex_Z2, &rwsem_Z2);
 183 }
 184
 185 /*
 186  * For spinlocks and rwlocks we also do hardirq-safe / softirq-safe tests.
 187  * The following functions use a lock from a simulated hardirq/softirq
 188  * context, causing the locks to be marked as hardirq-safe/softirq-safe:
 189  */
 190
 191 #define HARDIRQ_DISABLE         local_irq_disable
 192 #define HARDIRQ_ENABLE          local_irq_enable
 193
 194 #define HARDIRQ_ENTER()                         \
 195         local_irq_disable();                    \
 196         __irq_enter();                          \
 197         lockdep_hardirq_threaded();             \
 198         WARN_ON(!in_irq());
 199
 200 #define HARDIRQ_EXIT()                          \
 201         __irq_exit();                           \
 202         local_irq_enable();
 203
 204 #define SOFTIRQ_DISABLE         local_bh_disable
 205 #define SOFTIRQ_ENABLE          local_bh_enable
 206
 207 #define SOFTIRQ_ENTER()                         \
 208                 local_bh_disable();             \
 209                 local_irq_disable();            \
 210                 lockdep_softirq_enter();        \
 211                 WARN_ON(!in_softirq());
 212
 213 #define SOFTIRQ_EXIT()                          \
 214                 lockdep_softirq_exit();         \
 215                 local_irq_enable();             \
 216                 local_bh_enable();
 217
 218 /*
 219  * Shortcuts for lock/unlock API variants, to keep
 220  * the testcases compact:
 221  */
 222 #define L(x)                    spin_lock(&lock_##x)
 223 #define U(x)                    spin_unlock(&lock_##x)
 224 #define LU(x)                   L(x); U(x)
 225 #define SI(x)                   spin_lock_init(&lock_##x)
 226
 227 #define WL(x)                   write_lock(&rwlock_##x)
 228 #define WU(x)                   write_unlock(&rwlock_##x)
 229 #define WLU(x)                  WL(x); WU(x)
 230
 231 #define RL(x)                   read_lock(&rwlock_##x)
 232 #define RU(x)                   read_unlock(&rwlock_##x)
 233 #define RLU(x)                  RL(x); RU(x)
 234 #define RWI(x)                  rwlock_init(&rwlock_##x)
 235
 236 #define ML(x)                   mutex_lock(&mutex_##x)
 237 #define MU(x)                   mutex_unlock(&mutex_##x)
 238 #define MI(x)                   mutex_init(&mutex_##x)
 239
 240 #define RTL(x)                  rt_mutex_lock(&rtmutex_##x)
 241 #define RTU(x)                  rt_mutex_unlock(&rtmutex_##x)
 242 #define RTI(x)                  rt_mutex_init(&rtmutex_##x)
 243
 244 #define WSL(x)                  down_write(&rwsem_##x)
 245 #define WSU(x)                  up_write(&rwsem_##x)
 246
 247 #define RSL(x)                  down_read(&rwsem_##x)
 248 #define RSU(x)                  up_read(&rwsem_##x)
 249 #define RWSI(x)                 init_rwsem(&rwsem_##x)
 250
 251 #ifndef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
 252 #define WWAI(x)                 ww_acquire_init(x, &ww_lockdep)
 253 #else
 254 #define WWAI(x)                 do { ww_acquire_init(x, &ww_lockdep); (x)->deadlock_inject_countdown = ~0U; } while (0)
 255 #endif
 256 #define WWAD(x)                 ww_acquire_done(x)
 257 #define WWAF(x)                 ww_acquire_fini(x)
 258
 259 #define WWL(x, c)               ww_mutex_lock(x, c)
 260 #define WWT(x)                  ww_mutex_trylock(x)
 261 #define WWL1(x)                 ww_mutex_lock(x, NULL)
 262 #define WWU(x)                  ww_mutex_unlock(x)
 263
 264
 265 #define LOCK_UNLOCK_2(x,y)      LOCK(x); LOCK(y); UNLOCK(y); UNLOCK(x)
 266
 267 /*
 268  * Generate different permutations of the same testcase, using
 269  * the same basic lock-dependency/state events:
 270  */
 271
 272 #define GENERATE_TESTCASE(name)                 \
 273                                                 \
 274 static void name(void) { E(); }
 275
 276 #define GENERATE_PERMUTATIONS_2_EVENTS(name)    \
 277                                                 \
 278 static void name##_12(void) { E1(); E2(); }     \
 279 static void name##_21(void) { E2(); E1(); }
 280
 281 #define GENERATE_PERMUTATIONS_3_EVENTS(name)            \
 282                                                         \
 283 static void name##_123(void) { E1(); E2(); E3(); }      \
 284 static void name##_132(void) { E1(); E3(); E2(); }      \
 285 static void name##_213(void) { E2(); E1(); E3(); }      \
 286 static void name##_231(void) { E2(); E3(); E1(); }      \
 287 static void name##_312(void) { E3(); E1(); E2(); }      \
 288 static void name##_321(void) { E3(); E2(); E1(); }
 289
 290 /*
 291  * AA deadlock:
 292  */
 293
 294 #define E()                                     \
 295                                                 \
 296         LOCK(X1);                               \
 297         LOCK(X2); /* this one should fail */
 298
 299 /*
 300  * 6 testcases:
 301  */
 302 #include "locking-selftest-spin.h"
 303 GENERATE_TESTCASE(AA_spin)
 304 #include "locking-selftest-wlock.h"
 305 GENERATE_TESTCASE(AA_wlock)
 306 #include "locking-selftest-rlock.h"
 307 GENERATE_TESTCASE(AA_rlock)
 308 #include "locking-selftest-mutex.h"
 309 GENERATE_TESTCASE(AA_mutex)
 310 #include "locking-selftest-wsem.h"
 311 GENERATE_TESTCASE(AA_wsem)
 312 #include "locking-selftest-rsem.h"
 313 GENERATE_TESTCASE(AA_rsem)
 314
 315 #ifdef CONFIG_RT_MUTEXES
 316 #include "locking-selftest-rtmutex.h"
 317 GENERATE_TESTCASE(AA_rtmutex);
 318 #endif
 319
 320 #undef E
 321
 322 /*
 323  * Special-case for read-locking, they are
 324  * allowed to recurse on the same lock class:
 325  */
 326 static void rlock_AA1(void)
 327 {
 328         RL(X1);
 329         RL(X1); // this one should NOT fail
 330 }
 331
 332 static void rlock_AA1B(void)
 333 {
 334         RL(X1);
 335         RL(X2); // this one should NOT fail
 336 }
 337
 338 static void rsem_AA1(void)
 339 {
 340         RSL(X1);
 341         RSL(X1); // this one should fail
 342 }
 343
 344 static void rsem_AA1B(void)
 345 {
 346         RSL(X1);
 347         RSL(X2); // this one should fail
 348 }
 349 /*
 350  * The mixing of read and write locks is not allowed:
 351  */
 352 static void rlock_AA2(void)
 353 {
 354         RL(X1);
 355         WL(X2); // this one should fail
 356 }
 357
 358 static void rsem_AA2(void)
 359 {
 360         RSL(X1);
 361         WSL(X2); // this one should fail
 362 }
 363
 364 static void rlock_AA3(void)
 365 {
 366         WL(X1);
 367         RL(X2); // this one should fail
 368 }
 369
 370 static void rsem_AA3(void)
 371 {
 372         WSL(X1);
 373         RSL(X2); // this one should fail
 374 }
 375
 376 /*
 377  * read_lock(A)
 378  * spin_lock(B)
 379  *              spin_lock(B)
 380  *              write_lock(A)
 381  */
 382 static void rlock_ABBA1(void)
 383 {
 384         RL(X1);
 385         L(Y1);
 386         U(Y1);
 387         RU(X1);
 388
 389         L(Y1);
 390         WL(X1);
 391         WU(X1);
 392         U(Y1); // should fail
 393 }
 394
 395 static void rwsem_ABBA1(void)
 396 {
 397         RSL(X1);
 398         ML(Y1);
 399         MU(Y1);
 400         RSU(X1);
 401
 402         ML(Y1);
 403         WSL(X1);
 404         WSU(X1);
 405         MU(Y1); // should fail
 406 }
 407
 408 /*
 409  * read_lock(A)
 410  * spin_lock(B)
 411  *              spin_lock(B)
 412  *              write_lock(A)
 413  *
 414  * This test case is aimed at poking whether the chain cache prevents us from
 415  * detecting a read-lock/lock-write deadlock: if the chain cache doesn't differ
 416  * read/write locks, the following case may happen
 417  *
 418  *      { read_lock(A)->lock(B) dependency exists }
 419  *
 420  *      P0:
 421  *      lock(B);
 422  *      read_lock(A);
 423  *
 424  *      { Not a deadlock, B -> A is added in the chain cache }
 425  *
 426  *      P1:
 427  *      lock(B);
 428  *      write_lock(A);
 429  *
 430  *      { B->A found in chain cache, not reported as a deadlock }
 431  *
 432  */
 433 static void rlock_chaincache_ABBA1(void)
 434 {
 435         RL(X1);
 436         L(Y1);
 437         U(Y1);
 438         RU(X1);
 439
 440         L(Y1);
 441         RL(X1);
 442         RU(X1);
 443         U(Y1);
 444
 445         L(Y1);
 446         WL(X1);
 447         WU(X1);
 448         U(Y1); // should fail
 449 }
 450
 451 /*
 452  * read_lock(A)
 453  * spin_lock(B)
 454  *              spin_lock(B)
 455  *              read_lock(A)
 456  */
 457 static void rlock_ABBA2(void)
 458 {
 459         RL(X1);
 460         L(Y1);
 461         U(Y1);
 462         RU(X1);
 463
 464         L(Y1);
 465         RL(X1);
 466         RU(X1);
 467         U(Y1); // should NOT fail
 468 }
 469
 470 static void rwsem_ABBA2(void)
 471 {
 472         RSL(X1);
 473         ML(Y1);
 474         MU(Y1);
 475         RSU(X1);
 476
 477         ML(Y1);
 478         RSL(X1);
 479         RSU(X1);
 480         MU(Y1); // should fail
 481 }
 482
 483
 484 /*
 485  * write_lock(A)
 486  * spin_lock(B)
 487  *              spin_lock(B)
 488  *              write_lock(A)
 489  */
 490 static void rlock_ABBA3(void)
 491 {
 492         WL(X1);
 493         L(Y1);
 494         U(Y1);
 495         WU(X1);
 496
 497         L(Y1);
 498         WL(X1);
 499         WU(X1);
 500         U(Y1); // should fail
 501 }
 502
 503 static void rwsem_ABBA3(void)
 504 {
 505         WSL(X1);
 506         ML(Y1);
 507         MU(Y1);
 508         WSU(X1);
 509
 510         ML(Y1);
 511         WSL(X1);
 512         WSU(X1);
 513         MU(Y1); // should fail
 514 }
 515
 516 /*
 517  * ABBA deadlock:
 518  */
 519
 520 #define E()                                     \
 521                                                 \
 522         LOCK_UNLOCK_2(A, B);                    \
 523         LOCK_UNLOCK_2(B, A); /* fail */
 524
 525 /*
 526  * 6 testcases:
 527  */
 528 #include "locking-selftest-spin.h"
 529 GENERATE_TESTCASE(ABBA_spin)
 530 #include "locking-selftest-wlock.h"
 531 GENERATE_TESTCASE(ABBA_wlock)
 532 #include "locking-selftest-rlock.h"
 533 GENERATE_TESTCASE(ABBA_rlock)
 534 #include "locking-selftest-mutex.h"
 535 GENERATE_TESTCASE(ABBA_mutex)
 536 #include "locking-selftest-wsem.h"
 537 GENERATE_TESTCASE(ABBA_wsem)
 538 #include "locking-selftest-rsem.h"
 539 GENERATE_TESTCASE(ABBA_rsem)
 540
 541 #ifdef CONFIG_RT_MUTEXES
 542 #include "locking-selftest-rtmutex.h"
 543 GENERATE_TESTCASE(ABBA_rtmutex);
 544 #endif
 545
 546 #undef E
 547
 548 /*
 549  * AB BC CA deadlock:
 550  */
 551
 552 #define E()                                     \
 553                                                 \
 554         LOCK_UNLOCK_2(A, B);                    \
 555         LOCK_UNLOCK_2(B, C);                    \
 556         LOCK_UNLOCK_2(C, A); /* fail */
 557
 558 /*
 559  * 6 testcases:
 560  */
 561 #include "locking-selftest-spin.h"
 562 GENERATE_TESTCASE(ABBCCA_spin)
 563 #include "locking-selftest-wlock.h"
 564 GENERATE_TESTCASE(ABBCCA_wlock)
 565 #include "locking-selftest-rlock.h"
 566 GENERATE_TESTCASE(ABBCCA_rlock)
 567 #include "locking-selftest-mutex.h"
 568 GENERATE_TESTCASE(ABBCCA_mutex)
 569 #include "locking-selftest-wsem.h"
 570 GENERATE_TESTCASE(ABBCCA_wsem)
 571 #include "locking-selftest-rsem.h"
 572 GENERATE_TESTCASE(ABBCCA_rsem)
 573
 574 #ifdef CONFIG_RT_MUTEXES
 575 #include "locking-selftest-rtmutex.h"
 576 GENERATE_TESTCASE(ABBCCA_rtmutex);
 577 #endif
 578
 579 #undef E
 580
 581 /*
 582  * AB CA BC deadlock:
 583  */
 584
 585 #define E()                                     \
 586                                                 \
 587         LOCK_UNLOCK_2(A, B);                    \
 588         LOCK_UNLOCK_2(C, A);                    \
 589         LOCK_UNLOCK_2(B, C); /* fail */
 590
 591 /*
 592  * 6 testcases:
 593  */
 594 #include "locking-selftest-spin.h"
 595 GENERATE_TESTCASE(ABCABC_spin)
 596 #include "locking-selftest-wlock.h"
 597 GENERATE_TESTCASE(ABCABC_wlock)
 598 #include "locking-selftest-rlock.h"
 599 GENERATE_TESTCASE(ABCABC_rlock)
 600 #include "locking-selftest-mutex.h"
 601 GENERATE_TESTCASE(ABCABC_mutex)
 602 #include "locking-selftest-wsem.h"
 603 GENERATE_TESTCASE(ABCABC_wsem)
 604 #include "locking-selftest-rsem.h"
 605 GENERATE_TESTCASE(ABCABC_rsem)
 606
 607 #ifdef CONFIG_RT_MUTEXES
 608 #include "locking-selftest-rtmutex.h"
 609 GENERATE_TESTCASE(ABCABC_rtmutex);
 610 #endif
 611
 612 #undef E
 613
 614 /*
 615  * AB BC CD DA deadlock:
 616  */
 617
 618 #define E()                                     \
 619                                                 \
 620         LOCK_UNLOCK_2(A, B);                    \
 621         LOCK_UNLOCK_2(B, C);                    \
 622         LOCK_UNLOCK_2(C, D);                    \
 623         LOCK_UNLOCK_2(D, A); /* fail */
 624
 625 /*
 626  * 6 testcases:
 627  */
 628 #include "locking-selftest-spin.h"
 629 GENERATE_TESTCASE(ABBCCDDA_spin)
 630 #include "locking-selftest-wlock.h"
 631 GENERATE_TESTCASE(ABBCCDDA_wlock)
 632 #include "locking-selftest-rlock.h"
 633 GENERATE_TESTCASE(ABBCCDDA_rlock)
 634 #include "locking-selftest-mutex.h"
 635 GENERATE_TESTCASE(ABBCCDDA_mutex)
 636 #include "locking-selftest-wsem.h"
 637 GENERATE_TESTCASE(ABBCCDDA_wsem)
 638 #include "locking-selftest-rsem.h"
 639 GENERATE_TESTCASE(ABBCCDDA_rsem)
 640
 641 #ifdef CONFIG_RT_MUTEXES
 642 #include "locking-selftest-rtmutex.h"
 643 GENERATE_TESTCASE(ABBCCDDA_rtmutex);
 644 #endif
 645
 646 #undef E
 647
 648 /*
 649  * AB CD BD DA deadlock:
 650  */
 651 #define E()                                     \
 652                                                 \
 653         LOCK_UNLOCK_2(A, B);                    \
 654         LOCK_UNLOCK_2(C, D);                    \
 655         LOCK_UNLOCK_2(B, D);                    \
 656         LOCK_UNLOCK_2(D, A); /* fail */
 657
 658 /*
 659  * 6 testcases:
 660  */
 661 #include "locking-selftest-spin.h"
 662 GENERATE_TESTCASE(ABCDBDDA_spin)
 663 #include "locking-selftest-wlock.h"
 664 GENERATE_TESTCASE(ABCDBDDA_wlock)
 665 #include "locking-selftest-rlock.h"
 666 GENERATE_TESTCASE(ABCDBDDA_rlock)
 667 #include "locking-selftest-mutex.h"
 668 GENERATE_TESTCASE(ABCDBDDA_mutex)
 669 #include "locking-selftest-wsem.h"
 670 GENERATE_TESTCASE(ABCDBDDA_wsem)
 671 #include "locking-selftest-rsem.h"
 672 GENERATE_TESTCASE(ABCDBDDA_rsem)
 673
 674 #ifdef CONFIG_RT_MUTEXES
 675 #include "locking-selftest-rtmutex.h"
 676 GENERATE_TESTCASE(ABCDBDDA_rtmutex);
 677 #endif
 678
 679 #undef E
 680
 681 /*
 682  * AB CD BC DA deadlock:
 683  */
 684 #define E()                                     \
 685                                                 \
 686         LOCK_UNLOCK_2(A, B);                    \
 687         LOCK_UNLOCK_2(C, D);                    \
 688         LOCK_UNLOCK_2(B, C);                    \
 689         LOCK_UNLOCK_2(D, A); /* fail */
 690
 691 /*
 692  * 6 testcases:
 693  */
 694 #include "locking-selftest-spin.h"
 695 GENERATE_TESTCASE(ABCDBCDA_spin)
 696 #include "locking-selftest-wlock.h"
 697 GENERATE_TESTCASE(ABCDBCDA_wlock)
 698 #include "locking-selftest-rlock.h"
 699 GENERATE_TESTCASE(ABCDBCDA_rlock)
 700 #include "locking-selftest-mutex.h"
 701 GENERATE_TESTCASE(ABCDBCDA_mutex)
 702 #include "locking-selftest-wsem.h"
 703 GENERATE_TESTCASE(ABCDBCDA_wsem)
 704 #include "locking-selftest-rsem.h"
 705 GENERATE_TESTCASE(ABCDBCDA_rsem)
 706
 707 #ifdef CONFIG_RT_MUTEXES
 708 #include "locking-selftest-rtmutex.h"
 709 GENERATE_TESTCASE(ABCDBCDA_rtmutex);
 710 #endif
 711
 712 #undef E
 713
 714 /*
 715  * Double unlock:
 716  */
 717 #define E()                                     \
 718                                                 \
 719         LOCK(A);                                \
 720         UNLOCK(A);                              \
 721         UNLOCK(A); /* fail */
 722
 723 /*
 724  * 6 testcases:
 725  */
 726 #include "locking-selftest-spin.h"
 727 GENERATE_TESTCASE(double_unlock_spin)
 728 #include "locking-selftest-wlock.h"
 729 GENERATE_TESTCASE(double_unlock_wlock)
 730 #include "locking-selftest-rlock.h"
 731 GENERATE_TESTCASE(double_unlock_rlock)
 732 #include "locking-selftest-mutex.h"
 733 GENERATE_TESTCASE(double_unlock_mutex)
 734 #include "locking-selftest-wsem.h"
 735 GENERATE_TESTCASE(double_unlock_wsem)
 736 #include "locking-selftest-rsem.h"
 737 GENERATE_TESTCASE(double_unlock_rsem)
 738
 739 #ifdef CONFIG_RT_MUTEXES
 740 #include "locking-selftest-rtmutex.h"
 741 GENERATE_TESTCASE(double_unlock_rtmutex);
 742 #endif
 743
 744 #undef E
 745
 746 /*
 747  * initializing a held lock:
 748  */
 749 #define E()                                     \
 750                                                 \
 751         LOCK(A);                                \
 752         INIT(A); /* fail */
 753
 754 /*
 755  * 6 testcases:
 756  */
 757 #include "locking-selftest-spin.h"
 758 GENERATE_TESTCASE(init_held_spin)
 759 #include "locking-selftest-wlock.h"
 760 GENERATE_TESTCASE(init_held_wlock)
 761 #include "locking-selftest-rlock.h"
 762 GENERATE_TESTCASE(init_held_rlock)
 763 #include "locking-selftest-mutex.h"
 764 GENERATE_TESTCASE(init_held_mutex)
 765 #include "locking-selftest-wsem.h"
 766 GENERATE_TESTCASE(init_held_wsem)
 767 #include "locking-selftest-rsem.h"
 768 GENERATE_TESTCASE(init_held_rsem)
 769
 770 #ifdef CONFIG_RT_MUTEXES
 771 #include "locking-selftest-rtmutex.h"
 772 GENERATE_TESTCASE(init_held_rtmutex);
 773 #endif
 774
 775 #undef E
 776
 777 /*
 778  * locking an irq-safe lock with irqs enabled:
 779  */
 780 #define E1()                            \
 781                                         \
 782         IRQ_ENTER();                    \
 783         LOCK(A);                        \
 784         UNLOCK(A);                      \
 785         IRQ_EXIT();
 786
 787 #define E2()                            \
 788                                         \
 789         LOCK(A);                        \
 790         UNLOCK(A);
 791
 792 /*
 793  * Generate 24 testcases:
 794  */
 795 #include "locking-selftest-spin-hardirq.h"
 796 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin)
 797
 798 #include "locking-selftest-rlock-hardirq.h"
 799 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock)
 800
 801 #include "locking-selftest-wlock-hardirq.h"
 802 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_wlock)
 803
 804 #include "locking-selftest-spin-softirq.h"
 805 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_spin)
 806
 807 #include "locking-selftest-rlock-softirq.h"
 808 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_rlock)
 809
 810 #include "locking-selftest-wlock-softirq.h"
 811 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock)
 812
 813 #undef E1
 814 #undef E2
 815
 816 /*
 817  * Enabling hardirqs with a softirq-safe lock held:
 818  */
 819 #define E1()                            \
 820                                         \
 821         SOFTIRQ_ENTER();                \
 822         LOCK(A);                        \
 823         UNLOCK(A);                      \
 824         SOFTIRQ_EXIT();
 825
 826 #define E2()                            \
 827                                         \
 828         HARDIRQ_DISABLE();              \
 829         LOCK(A);                        \
 830         HARDIRQ_ENABLE();               \
 831         UNLOCK(A);
 832
 833 /*
 834  * Generate 12 testcases:
 835  */
 836 #include "locking-selftest-spin.h"
 837 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_spin)
 838
 839 #include "locking-selftest-wlock.h"
 840 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_wlock)
 841
 842 #include "locking-selftest-rlock.h"
 843 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_rlock)
 844
 845 #undef E1
 846 #undef E2
 847
 848 /*
 849  * Enabling irqs with an irq-safe lock held:
 850  */
 851 #define E1()                            \
 852                                         \
 853         IRQ_ENTER();                    \
 854         LOCK(A);                        \
 855         UNLOCK(A);                      \
 856         IRQ_EXIT();
 857
 858 #define E2()                            \
 859                                         \
 860         IRQ_DISABLE();                  \
 861         LOCK(A);                        \
 862         IRQ_ENABLE();                   \
 863         UNLOCK(A);
 864
 865 /*
 866  * Generate 24 testcases:
 867  */
 868 #include "locking-selftest-spin-hardirq.h"
 869 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin)
 870
 871 #include "locking-selftest-rlock-hardirq.h"
 872 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock)
 873
 874 #include "locking-selftest-wlock-hardirq.h"
 875 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_wlock)
 876
 877 #include "locking-selftest-spin-softirq.h"
 878 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_spin)
 879
 880 #include "locking-selftest-rlock-softirq.h"
 881 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_rlock)
 882
 883 #include "locking-selftest-wlock-softirq.h"
 884 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock)
 885
 886 #undef E1
 887 #undef E2
 888
 889 /*
 890  * Acquiring a irq-unsafe lock while holding an irq-safe-lock:
 891  */
 892 #define E1()                            \
 893                                         \
 894         LOCK(A);                        \
 895         LOCK(B);                        \
 896         UNLOCK(B);                      \
 897         UNLOCK(A);                      \
 898
 899 #define E2()                            \
 900                                         \
 901         LOCK(B);                        \
 902         UNLOCK(B);
 903
 904 #define E3()                            \
 905                                         \
 906         IRQ_ENTER();                    \
 907         LOCK(A);                        \
 908         UNLOCK(A);                      \
 909         IRQ_EXIT();
 910
 911 /*
 912  * Generate 36 testcases:
 913  */
 914 #include "locking-selftest-spin-hardirq.h"
 915 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin)
 916
 917 #include "locking-selftest-rlock-hardirq.h"
 918 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock)
 919
 920 #include "locking-selftest-wlock-hardirq.h"
 921 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_wlock)
 922
 923 #include "locking-selftest-spin-softirq.h"
 924 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_spin)
 925
 926 #include "locking-selftest-rlock-softirq.h"
 927 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_rlock)
 928
 929 #include "locking-selftest-wlock-softirq.h"
 930 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock)
 931
 932 #undef E1
 933 #undef E2
 934 #undef E3
 935
 936 /*
 937  * If a lock turns into softirq-safe, but earlier it took
 938  * a softirq-unsafe lock:
 939  */
 940
 941 #define E1()                            \
 942         IRQ_DISABLE();                  \
 943         LOCK(A);                        \
 944         LOCK(B);                        \
 945         UNLOCK(B);                      \
 946         UNLOCK(A);                      \
 947         IRQ_ENABLE();
 948
 949 #define E2()                            \
 950         LOCK(B);                        \
 951         UNLOCK(B);
 952
 953 #define E3()                            \
 954         IRQ_ENTER();                    \
 955         LOCK(A);                        \
 956         UNLOCK(A);                      \
 957         IRQ_EXIT();
 958
 959 /*
 960  * Generate 36 testcases:
 961  */
 962 #include "locking-selftest-spin-hardirq.h"
 963 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin)
 964
 965 #include "locking-selftest-rlock-hardirq.h"
 966 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock)
 967
 968 #include "locking-selftest-wlock-hardirq.h"
 969 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_wlock)
 970
 971 #include "locking-selftest-spin-softirq.h"
 972 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_spin)
 973
 974 #include "locking-selftest-rlock-softirq.h"
 975 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_rlock)
 976
 977 #include "locking-selftest-wlock-softirq.h"
 978 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock)
 979
 980 #undef E1
 981 #undef E2
 982 #undef E3
 983
 984 /*
 985  * read-lock / write-lock irq inversion.
 986  *
 987  * Deadlock scenario:
 988  *
 989  * CPU#1 is at #1, i.e. it has write-locked A, but has not
 990  * taken B yet.
 991  *
 992  * CPU#2 is at #2, i.e. it has locked B.
 993  *
 994  * Hardirq hits CPU#2 at point #2 and is trying to read-lock A.
 995  *
 996  * The deadlock occurs because CPU#1 will spin on B, and CPU#2
 997  * will spin on A.
 998  */
 999
1000 #define E1()                            \
1001                                         \
1002         IRQ_DISABLE();                  \
1003         WL(A);                          \
1004         LOCK(B);                        \
1005         UNLOCK(B);                      \
1006         WU(A);                          \
1007         IRQ_ENABLE();
1008
1009 #define E2()                            \
1010                                         \
1011         LOCK(B);                        \
1012         UNLOCK(B);
1013
1014 #define E3()                            \
1015                                         \
1016         IRQ_ENTER();                    \
1017         RL(A);                          \
1018         RU(A);                          \
1019         IRQ_EXIT();
1020
1021 /*
1022  * Generate 36 testcases:
1023  */
1024 #include "locking-selftest-spin-hardirq.h"
1025 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_spin)
1026
1027 #include "locking-selftest-rlock-hardirq.h"
1028 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_rlock)
1029
1030 #include "locking-selftest-wlock-hardirq.h"
1031 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_wlock)
1032
1033 #include "locking-selftest-spin-softirq.h"
1034 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_spin)
1035
1036 #include "locking-selftest-rlock-softirq.h"
1037 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_rlock)
1038
1039 #include "locking-selftest-wlock-softirq.h"
1040 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_wlock)
1041
1042 #undef E1
1043 #undef E2
1044 #undef E3
1045
1046 /*
1047  * write-read / write-read / write-read deadlock even if read is recursive
1048  */
1049
1050 #define E1()                            \
1051                                         \
1052         WL(X1);                         \
1053         RL(Y1);                         \
1054         RU(Y1);                         \
1055         WU(X1);
1056
1057 #define E2()                            \
1058                                         \
1059         WL(Y1);                         \
1060         RL(Z1);                         \
1061         RU(Z1);                         \
1062         WU(Y1);
1063
1064 #define E3()                            \
1065                                         \
1066         WL(Z1);                         \
1067         RL(X1);                         \
1068         RU(X1);                         \
1069         WU(Z1);
1070
1071 #include "locking-selftest-rlock.h"
1072 GENERATE_PERMUTATIONS_3_EVENTS(W1R2_W2R3_W3R1)
1073
1074 #undef E1
1075 #undef E2
1076 #undef E3
1077
1078 /*
1079  * write-write / read-read / write-read deadlock even if read is recursive
1080  */
1081
1082 #define E1()                            \
1083                                         \
1084         WL(X1);                         \
1085         WL(Y1);                         \
1086         WU(Y1);                         \
1087         WU(X1);
1088
1089 #define E2()                            \
1090                                         \
1091         RL(Y1);                         \
1092         RL(Z1);                         \
1093         RU(Z1);                         \
1094         RU(Y1);
1095
1096 #define E3()                            \
1097                                         \
1098         WL(Z1);                         \
1099         RL(X1);                         \
1100         RU(X1);                         \
1101         WU(Z1);
1102
1103 #include "locking-selftest-rlock.h"
1104 GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_W3R1)
1105
1106 #undef E1
1107 #undef E2
1108 #undef E3
1109
1110 /*
1111  * write-write / read-read / read-write is not deadlock when read is recursive
1112  */
1113
1114 #define E1()                            \
1115                                         \
1116         WL(X1);                         \
1117         WL(Y1);                         \
1118         WU(Y1);                         \
1119         WU(X1);
1120
1121 #define E2()                            \
1122                                         \
1123         RL(Y1);                         \
1124         RL(Z1);                         \
1125         RU(Z1);                         \
1126         RU(Y1);
1127
1128 #define E3()                            \
1129                                         \
1130         RL(Z1);                         \
1131         WL(X1);                         \
1132         WU(X1);                         \
1133         RU(Z1);
1134
1135 #include "locking-selftest-rlock.h"
1136 GENERATE_PERMUTATIONS_3_EVENTS(W1R2_R2R3_W3W1)
1137
1138 #undef E1
1139 #undef E2
1140 #undef E3
1141
1142 /*
1143  * write-read / read-read / write-write is not deadlock when read is recursive
1144  */
1145
1146 #define E1()                            \
1147                                         \
1148         WL(X1);                         \
1149         RL(Y1);                         \
1150         RU(Y1);                         \
1151         WU(X1);
1152
1153 #define E2()                            \
1154                                         \
1155         RL(Y1);                         \
1156         RL(Z1);                         \
1157         RU(Z1);                         \
1158         RU(Y1);
1159
1160 #define E3()                            \
1161                                         \
1162         WL(Z1);                         \
1163         WL(X1);                         \
1164         WU(X1);                         \
1165         WU(Z1);
1166
1167 #include "locking-selftest-rlock.h"
1168 GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_R3W1)
1169
1170 #undef E1
1171 #undef E2
1172 #undef E3
1173 /*
1174  * read-lock / write-lock recursion that is actually safe.
1175  */
1176
1177 #define E1()                            \
1178                                         \
1179         IRQ_DISABLE();                  \
1180         WL(A);                          \
1181         WU(A);                          \
1182         IRQ_ENABLE();
1183
1184 #define E2()                            \
1185                                         \
1186         RL(A);                          \
1187         RU(A);                          \
1188
1189 #define E3()                            \
1190                                         \
1191         IRQ_ENTER();                    \
1192         LOCK(A);                        \
1193         L(B);                           \
1194         U(B);                           \
1195         UNLOCK(A);                      \
1196         IRQ_EXIT();
1197
1198 /*
1199  * Generate 24 testcases:
1200  */
1201 #include "locking-selftest-hardirq.h"
1202 #include "locking-selftest-rlock.h"
1203 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_rlock)
1204
1205 #include "locking-selftest-wlock.h"
1206 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_wlock)
1207
1208 #include "locking-selftest-softirq.h"
1209 #include "locking-selftest-rlock.h"
1210 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_rlock)
1211
1212 #include "locking-selftest-wlock.h"
1213 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_wlock)
1214
1215 #undef E1
1216 #undef E2
1217 #undef E3
1218
1219 /*
1220  * read-lock / write-lock recursion that is unsafe.
1221  */
1222
1223 #define E1()                            \
1224                                         \
1225         IRQ_DISABLE();                  \
1226         L(B);                           \
1227         LOCK(A);                        \
1228         UNLOCK(A);                      \
1229         U(B);                           \
1230         IRQ_ENABLE();
1231
1232 #define E2()                            \
1233                                         \
1234         RL(A);                          \
1235         RU(A);                          \
1236
1237 #define E3()                            \
1238                                         \
1239         IRQ_ENTER();                    \
1240         L(B);                           \
1241         U(B);                           \
1242         IRQ_EXIT();
1243
1244 /*
1245  * Generate 24 testcases:
1246  */
1247 #include "locking-selftest-hardirq.h"
1248 #include "locking-selftest-rlock.h"
1249 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_rlock)
1250
1251 #include "locking-selftest-wlock.h"
1252 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_wlock)
1253
1254 #include "locking-selftest-softirq.h"
1255 #include "locking-selftest-rlock.h"
1256 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_rlock)
1257
1258 #include "locking-selftest-wlock.h"
1259 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_wlock)
1260
1261 #undef E1
1262 #undef E2
1263 #undef E3
1264 /*
1265  * read-lock / write-lock recursion that is unsafe.
1266  *
1267  * A is a ENABLED_*_READ lock
1268  * B is a USED_IN_*_READ lock
1269  *
1270  * read_lock(A);
1271  *                      write_lock(B);
1272  * <interrupt>
1273  * read_lock(B);
1274  *                      write_lock(A); // if this one is read_lock(), no deadlock
1275  */
1276
1277 #define E1()                            \
1278                                         \
1279         IRQ_DISABLE();                  \
1280         WL(B);                          \
1281         LOCK(A);                        \
1282         UNLOCK(A);                      \
1283         WU(B);                          \
1284         IRQ_ENABLE();
1285
1286 #define E2()                            \
1287                                         \
1288         RL(A);                          \
1289         RU(A);                          \
1290
1291 #define E3()                            \
1292                                         \
1293         IRQ_ENTER();                    \
1294         RL(B);                          \
1295         RU(B);                          \
1296         IRQ_EXIT();
1297
1298 /*
1299  * Generate 24 testcases:
1300  */
1301 #include "locking-selftest-hardirq.h"
1302 #include "locking-selftest-rlock.h"
1303 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_rlock)
1304
1305 #include "locking-selftest-wlock.h"
1306 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_wlock)
1307
1308 #include "locking-selftest-softirq.h"
1309 #include "locking-selftest-rlock.h"
1310 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_rlock)
1311
1312 #include "locking-selftest-wlock.h"
1313 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_wlock)
1314
1315 #ifdef CONFIG_DEBUG_LOCK_ALLOC
1316 # define I_SPINLOCK(x)  lockdep_reset_lock(&lock_##x.dep_map)
1317 # define I_RAW_SPINLOCK(x)      lockdep_reset_lock(&raw_lock_##x.dep_map)
1318 # define I_RWLOCK(x)    lockdep_reset_lock(&rwlock_##x.dep_map)
1319 # define I_MUTEX(x)     lockdep_reset_lock(&mutex_##x.dep_map)
1320 # define I_RWSEM(x)     lockdep_reset_lock(&rwsem_##x.dep_map)
1321 # define I_WW(x)        lockdep_reset_lock(&x.dep_map)
1322 # define I_LOCAL_LOCK(x) lockdep_reset_lock(&local_##x.dep_map)
1323 #ifdef CONFIG_RT_MUTEXES
1324 # define I_RTMUTEX(x)   lockdep_reset_lock(&rtmutex_##x.dep_map)
1325 #endif
1326 #else
1327 # define I_SPINLOCK(x)
1328 # define I_RAW_SPINLOCK(x)
1329 # define I_RWLOCK(x)
1330 # define I_MUTEX(x)
1331 # define I_RWSEM(x)
1332 # define I_WW(x)
1333 # define I_LOCAL_LOCK(x)
1334 #endif
1335
1336 #ifndef I_RTMUTEX
1337 # define I_RTMUTEX(x)
1338 #endif
1339
1340 #ifdef CONFIG_RT_MUTEXES
1341 #define I2_RTMUTEX(x)   rt_mutex_init(&rtmutex_##x)
1342 #else
1343 #define I2_RTMUTEX(x)
1344 #endif
1345
1346 #define I1(x)                                   \
1347         do {                                    \
1348                 I_SPINLOCK(x);                  \
1349                 I_RWLOCK(x);                    \
1350                 I_MUTEX(x);                     \
1351                 I_RWSEM(x);                     \
1352                 I_RTMUTEX(x);                   \
1353         } while (0)
1354
1355 #define I2(x)                                   \
1356         do {                                    \
1357                 spin_lock_init(&lock_##x);      \
1358                 rwlock_init(&rwlock_##x);       \
1359                 mutex_init(&mutex_##x);         \
1360                 init_rwsem(&rwsem_##x);         \
1361                 I2_RTMUTEX(x);                  \
1362         } while (0)
1363
1364 static void reset_locks(void)
1365 {
1366         local_irq_disable();
1367         lockdep_free_key_range(&ww_lockdep.acquire_key, 1);
1368         lockdep_free_key_range(&ww_lockdep.mutex_key, 1);
1369
1370         I1(A); I1(B); I1(C); I1(D);
1371         I1(X1); I1(X2); I1(Y1); I1(Y2); I1(Z1); I1(Z2);
1372         I_WW(t); I_WW(t2); I_WW(o.base); I_WW(o2.base); I_WW(o3.base);
1373         I_RAW_SPINLOCK(A); I_RAW_SPINLOCK(B);
1374         I_LOCAL_LOCK(A);
1375
1376         lockdep_reset();
1377
1378         I2(A); I2(B); I2(C); I2(D);
1379         init_shared_classes();
1380         raw_spin_lock_init(&raw_lock_A);
1381         raw_spin_lock_init(&raw_lock_B);
1382         local_lock_init(&local_A);
1383
1384         ww_mutex_init(&o, &ww_lockdep); ww_mutex_init(&o2, &ww_lockdep); ww_mutex_init(&o3, &ww_lockdep);
1385         memset(&t, 0, sizeof(t)); memset(&t2, 0, sizeof(t2));
1386         memset(&ww_lockdep.acquire_key, 0, sizeof(ww_lockdep.acquire_key));
1387         memset(&ww_lockdep.mutex_key, 0, sizeof(ww_lockdep.mutex_key));
1388         local_irq_enable();
1389 }
1390
1391 #undef I
1392
1393 static int testcase_total;
1394 static int testcase_successes;
1395 static int expected_testcase_failures;
1396 static int unexpected_testcase_failures;
1397
1398 static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
1399 {
1400         unsigned long saved_preempt_count = preempt_count();
1401
1402         WARN_ON(irqs_disabled());
1403
1404         debug_locks_silent = !(debug_locks_verbose & lockclass_mask);
1405
1406         testcase_fn();
1407         /*
1408          * Filter out expected failures:
1409          */
1410 #ifndef CONFIG_PROVE_LOCKING
1411         if (expected == FAILURE && debug_locks) {
1412                 expected_testcase_failures++;
1413                 pr_cont("failed|");
1414         }
1415         else
1416 #endif
1417         if (debug_locks != expected) {
1418                 unexpected_testcase_failures++;
1419                 pr_cont("FAILED|");
1420         } else {
1421                 testcase_successes++;
1422                 pr_cont("  ok  |");
1423         }
1424         testcase_total++;
1425
1426         if (debug_locks_verbose & lockclass_mask)
1427                 pr_cont(" lockclass mask: %x, debug_locks: %d, expected: %d\n",
1428                         lockclass_mask, debug_locks, expected);
1429         /*
1430          * Some tests (e.g. double-unlock) might corrupt the preemption
1431          * count, so restore it:
1432          */
1433         preempt_count_set(saved_preempt_count);
1434 #ifdef CONFIG_TRACE_IRQFLAGS
1435         if (softirq_count())
1436                 current->softirqs_enabled = 0;
1437         else
1438                 current->softirqs_enabled = 1;
1439 #endif
1440
1441         reset_locks();
1442 }
1443
1444 #ifdef CONFIG_RT_MUTEXES
1445 #define dotest_rt(fn, e, m)     dotest((fn), (e), (m))
1446 #else
1447 #define dotest_rt(fn, e, m)
1448 #endif
1449
1450 static inline void print_testname(const char *testname)
1451 {
1452         printk("%33s:", testname);
1453 }
1454
1455 #define DO_TESTCASE_1(desc, name, nr)                           \
1456         print_testname(desc"/"#nr);                             \
1457         dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK);          \
1458         pr_cont("\n");
1459
1460 #define DO_TESTCASE_1B(desc, name, nr)                          \
1461         print_testname(desc"/"#nr);                             \
1462         dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK);          \
1463         pr_cont("\n");
1464
1465 #define DO_TESTCASE_1RR(desc, name, nr)                         \
1466         print_testname(desc"/"#nr);                             \
1467         pr_cont("             |");                              \
1468         dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK);          \
1469         pr_cont("\n");
1470
1471 #define DO_TESTCASE_1RRB(desc, name, nr)                        \
1472         print_testname(desc"/"#nr);                             \
1473         pr_cont("             |");                              \
1474         dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK);          \
1475         pr_cont("\n");
1476
1477
1478 #define DO_TESTCASE_3(desc, name, nr)                           \
1479         print_testname(desc"/"#nr);                             \
1480         dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN);       \
1481         dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK);    \
1482         dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK);    \
1483         pr_cont("\n");
1484
1485 #define DO_TESTCASE_3RW(desc, name, nr)                         \
1486         print_testname(desc"/"#nr);                             \
1487         dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN|LOCKTYPE_RWLOCK);\
1488         dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK);    \
1489         dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK);    \
1490         pr_cont("\n");
1491
1492 #define DO_TESTCASE_2RW(desc, name, nr)                         \
1493         print_testname(desc"/"#nr);                             \
1494         pr_cont("      |");                                     \
1495         dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK);    \
1496         dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK);    \
1497         pr_cont("\n");
1498
1499 #define DO_TESTCASE_2x2RW(desc, name, nr)                       \
1500         DO_TESTCASE_2RW("hard-"desc, name##_hard, nr)           \
1501         DO_TESTCASE_2RW("soft-"desc, name##_soft, nr)           \
1502
1503 #define DO_TESTCASE_6x2x2RW(desc, name)                         \
1504         DO_TESTCASE_2x2RW(desc, name, 123);                     \
1505         DO_TESTCASE_2x2RW(desc, name, 132);                     \
1506         DO_TESTCASE_2x2RW(desc, name, 213);                     \
1507         DO_TESTCASE_2x2RW(desc, name, 231);                     \
1508         DO_TESTCASE_2x2RW(desc, name, 312);                     \
1509         DO_TESTCASE_2x2RW(desc, name, 321);
1510
1511 #define DO_TESTCASE_6(desc, name)                               \
1512         print_testname(desc);                                   \
1513         dotest(name##_spin, FAILURE, LOCKTYPE_SPIN);            \
1514         dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK);         \
1515         dotest(name##_rlock, FAILURE, LOCKTYPE_RWLOCK);         \
1516         dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX);          \
1517         dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM);           \
1518         dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM);           \
1519         dotest_rt(name##_rtmutex, FAILURE, LOCKTYPE_RTMUTEX);   \
1520         pr_cont("\n");
1521
1522 #define DO_TESTCASE_6_SUCCESS(desc, name)                       \
1523         print_testname(desc);                                   \
1524         dotest(name##_spin, SUCCESS, LOCKTYPE_SPIN);            \
1525         dotest(name##_wlock, SUCCESS, LOCKTYPE_RWLOCK);         \
1526         dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK);         \
1527         dotest(name##_mutex, SUCCESS, LOCKTYPE_MUTEX);          \
1528         dotest(name##_wsem, SUCCESS, LOCKTYPE_RWSEM);           \
1529         dotest(name##_rsem, SUCCESS, LOCKTYPE_RWSEM);           \
1530         dotest_rt(name##_rtmutex, SUCCESS, LOCKTYPE_RTMUTEX);   \
1531         pr_cont("\n");
1532
1533 /*
1534  * 'read' variant: rlocks must not trigger.
1535  */
1536 #define DO_TESTCASE_6R(desc, name)                              \
1537         print_testname(desc);                                   \
1538         dotest(name##_spin, FAILURE, LOCKTYPE_SPIN);            \
1539         dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK);         \
1540         dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK);         \
1541         dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX);          \
1542         dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM);           \
1543         dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM);           \
1544         dotest_rt(name##_rtmutex, FAILURE, LOCKTYPE_RTMUTEX);   \
1545         pr_cont("\n");
1546
1547 #define DO_TESTCASE_2I(desc, name, nr)                          \
1548         DO_TESTCASE_1("hard-"desc, name##_hard, nr);            \
1549         DO_TESTCASE_1("soft-"desc, name##_soft, nr);
1550
1551 #define DO_TESTCASE_2IB(desc, name, nr)                         \
1552         DO_TESTCASE_1B("hard-"desc, name##_hard, nr);           \
1553         DO_TESTCASE_1B("soft-"desc, name##_soft, nr);
1554
1555 #define DO_TESTCASE_6I(desc, name, nr)                          \
1556         DO_TESTCASE_3("hard-"desc, name##_hard, nr);            \
1557         DO_TESTCASE_3("soft-"desc, name##_soft, nr);
1558
1559 #define DO_TESTCASE_6IRW(desc, name, nr)                        \
1560         DO_TESTCASE_3RW("hard-"desc, name##_hard, nr);          \
1561         DO_TESTCASE_3RW("soft-"desc, name##_soft, nr);
1562
1563 #define DO_TESTCASE_2x3(desc, name)                             \
1564         DO_TESTCASE_3(desc, name, 12);                          \
1565         DO_TESTCASE_3(desc, name, 21);
1566
1567 #define DO_TESTCASE_2x6(desc, name)                             \
1568         DO_TESTCASE_6I(desc, name, 12);                         \
1569         DO_TESTCASE_6I(desc, name, 21);
1570
1571 #define DO_TESTCASE_6x2(desc, name)                             \
1572         DO_TESTCASE_2I(desc, name, 123);                        \
1573         DO_TESTCASE_2I(desc, name, 132);                        \
1574         DO_TESTCASE_2I(desc, name, 213);                        \
1575         DO_TESTCASE_2I(desc, name, 231);                        \
1576         DO_TESTCASE_2I(desc, name, 312);                        \
1577         DO_TESTCASE_2I(desc, name, 321);
1578
1579 #define DO_TESTCASE_6x2B(desc, name)                            \
1580         DO_TESTCASE_2IB(desc, name, 123);                       \
1581         DO_TESTCASE_2IB(desc, name, 132);                       \
1582         DO_TESTCASE_2IB(desc, name, 213);                       \
1583         DO_TESTCASE_2IB(desc, name, 231);                       \
1584         DO_TESTCASE_2IB(desc, name, 312);                       \
1585         DO_TESTCASE_2IB(desc, name, 321);
1586
1587 #define DO_TESTCASE_6x1RR(desc, name)                           \
1588         DO_TESTCASE_1RR(desc, name, 123);                       \
1589         DO_TESTCASE_1RR(desc, name, 132);                       \
1590         DO_TESTCASE_1RR(desc, name, 213);                       \
1591         DO_TESTCASE_1RR(desc, name, 231);                       \
1592         DO_TESTCASE_1RR(desc, name, 312);                       \
1593         DO_TESTCASE_1RR(desc, name, 321);
1594
1595 #define DO_TESTCASE_6x1RRB(desc, name)                          \
1596         DO_TESTCASE_1RRB(desc, name, 123);                      \
1597         DO_TESTCASE_1RRB(desc, name, 132);                      \
1598         DO_TESTCASE_1RRB(desc, name, 213);                      \
1599         DO_TESTCASE_1RRB(desc, name, 231);                      \
1600         DO_TESTCASE_1RRB(desc, name, 312);                      \
1601         DO_TESTCASE_1RRB(desc, name, 321);
1602
1603 #define DO_TESTCASE_6x6(desc, name)                             \
1604         DO_TESTCASE_6I(desc, name, 123);                        \
1605         DO_TESTCASE_6I(desc, name, 132);                        \
1606         DO_TESTCASE_6I(desc, name, 213);                        \
1607         DO_TESTCASE_6I(desc, name, 231);                        \
1608         DO_TESTCASE_6I(desc, name, 312);                        \
1609         DO_TESTCASE_6I(desc, name, 321);
1610
1611 #define DO_TESTCASE_6x6RW(desc, name)                           \
1612         DO_TESTCASE_6IRW(desc, name, 123);                      \
1613         DO_TESTCASE_6IRW(desc, name, 132);                      \
1614         DO_TESTCASE_6IRW(desc, name, 213);                      \
1615         DO_TESTCASE_6IRW(desc, name, 231);                      \
1616         DO_TESTCASE_6IRW(desc, name, 312);                      \
1617         DO_TESTCASE_6IRW(desc, name, 321);
1618
1619 static void ww_test_fail_acquire(void)
1620 {
1621         int ret;
1622
1623         WWAI(&t);
1624         t.stamp++;
1625
1626         ret = WWL(&o, &t);
1627
1628         if (WARN_ON(!o.ctx) ||
1629             WARN_ON(ret))
1630                 return;
1631
1632         /* No lockdep test, pure API */
1633         ret = WWL(&o, &t);
1634         WARN_ON(ret != -EALREADY);
1635
1636         ret = WWT(&o);
1637         WARN_ON(ret);
1638
1639         t2 = t;
1640         t2.stamp++;
1641         ret = WWL(&o, &t2);
1642         WARN_ON(ret != -EDEADLK);
1643         WWU(&o);
1644
1645         if (WWT(&o))
1646                 WWU(&o);
1647 #ifdef CONFIG_DEBUG_LOCK_ALLOC
1648         else
1649                 DEBUG_LOCKS_WARN_ON(1);
1650 #endif
1651 }
1652
1653 static void ww_test_normal(void)
1654 {
1655         int ret;
1656
1657         WWAI(&t);
1658
1659         /*
1660          * None of the ww_mutex codepaths should be taken in the 'normal'
1661          * mutex calls. The easiest way to verify this is by using the
1662          * normal mutex calls, and making sure o.ctx is unmodified.
1663          */
1664
1665         /* mutex_lock (and indirectly, mutex_lock_nested) */
1666         o.ctx = (void *)~0UL;
1667         mutex_lock(&o.base);
1668         mutex_unlock(&o.base);
1669         WARN_ON(o.ctx != (void *)~0UL);
1670
1671         /* mutex_lock_interruptible (and *_nested) */
1672         o.ctx = (void *)~0UL;
1673         ret = mutex_lock_interruptible(&o.base);
1674         if (!ret)
1675                 mutex_unlock(&o.base);
1676         else
1677                 WARN_ON(1);
1678         WARN_ON(o.ctx != (void *)~0UL);
1679
1680         /* mutex_lock_killable (and *_nested) */
1681         o.ctx = (void *)~0UL;
1682         ret = mutex_lock_killable(&o.base);
1683         if (!ret)
1684                 mutex_unlock(&o.base);
1685         else
1686                 WARN_ON(1);
1687         WARN_ON(o.ctx != (void *)~0UL);
1688
1689         /* trylock, succeeding */
1690         o.ctx = (void *)~0UL;
1691         ret = mutex_trylock(&o.base);
1692         WARN_ON(!ret);
1693         if (ret)
1694                 mutex_unlock(&o.base);
1695         else
1696                 WARN_ON(1);
1697         WARN_ON(o.ctx != (void *)~0UL);
1698
1699         /* trylock, failing */
1700         o.ctx = (void *)~0UL;
1701         mutex_lock(&o.base);
1702         ret = mutex_trylock(&o.base);
1703         WARN_ON(ret);
1704         mutex_unlock(&o.base);
1705         WARN_ON(o.ctx != (void *)~0UL);
1706
1707         /* nest_lock */
1708         o.ctx = (void *)~0UL;
1709         mutex_lock_nest_lock(&o.base, &t);
1710         mutex_unlock(&o.base);
1711         WARN_ON(o.ctx != (void *)~0UL);
1712 }
1713
1714 static void ww_test_two_contexts(void)
1715 {
1716         WWAI(&t);
1717         WWAI(&t2);
1718 }
1719
1720 static void ww_test_diff_class(void)
1721 {
1722         WWAI(&t);
1723 #ifdef CONFIG_DEBUG_MUTEXES
1724         t.ww_class = NULL;
1725 #endif
1726         WWL(&o, &t);
1727 }
1728
1729 static void ww_test_context_done_twice(void)
1730 {
1731         WWAI(&t);
1732         WWAD(&t);
1733         WWAD(&t);
1734         WWAF(&t);
1735 }
1736
1737 static void ww_test_context_unlock_twice(void)
1738 {
1739         WWAI(&t);
1740         WWAD(&t);
1741         WWAF(&t);
1742         WWAF(&t);
1743 }
1744
1745 static void ww_test_context_fini_early(void)
1746 {
1747         WWAI(&t);
1748         WWL(&o, &t);
1749         WWAD(&t);
1750         WWAF(&t);
1751 }
1752
1753 static void ww_test_context_lock_after_done(void)
1754 {
1755         WWAI(&t);
1756         WWAD(&t);
1757         WWL(&o, &t);
1758 }
1759
1760 static void ww_test_object_unlock_twice(void)
1761 {
1762         WWL1(&o);
1763         WWU(&o);
1764         WWU(&o);
1765 }
1766
1767 static void ww_test_object_lock_unbalanced(void)
1768 {
1769         WWAI(&t);
1770         WWL(&o, &t);
1771         t.acquired = 0;
1772         WWU(&o);
1773         WWAF(&t);
1774 }
1775
1776 static void ww_test_object_lock_stale_context(void)
1777 {
1778         WWAI(&t);
1779         o.ctx = &t2;
1780         WWL(&o, &t);
1781 }
1782
1783 static void ww_test_edeadlk_normal(void)
1784 {
1785         int ret;
1786
1787         mutex_lock(&o2.base);
1788         o2.ctx = &t2;
1789         mutex_release(&o2.base.dep_map, _THIS_IP_);
1790
1791         WWAI(&t);
1792         t2 = t;
1793         t2.stamp--;
1794
1795         ret = WWL(&o, &t);
1796         WARN_ON(ret);
1797
1798         ret = WWL(&o2, &t);
1799         WARN_ON(ret != -EDEADLK);
1800
1801         o2.ctx = NULL;
1802         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1803         mutex_unlock(&o2.base);
1804         WWU(&o);
1805
1806         WWL(&o2, &t);
1807 }
1808
1809 static void ww_test_edeadlk_normal_slow(void)
1810 {
1811         int ret;
1812
1813         mutex_lock(&o2.base);
1814         mutex_release(&o2.base.dep_map, _THIS_IP_);
1815         o2.ctx = &t2;
1816
1817         WWAI(&t);
1818         t2 = t;
1819         t2.stamp--;
1820
1821         ret = WWL(&o, &t);
1822         WARN_ON(ret);
1823
1824         ret = WWL(&o2, &t);
1825         WARN_ON(ret != -EDEADLK);
1826
1827         o2.ctx = NULL;
1828         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1829         mutex_unlock(&o2.base);
1830         WWU(&o);
1831
1832         ww_mutex_lock_slow(&o2, &t);
1833 }
1834
1835 static void ww_test_edeadlk_no_unlock(void)
1836 {
1837         int ret;
1838
1839         mutex_lock(&o2.base);
1840         o2.ctx = &t2;
1841         mutex_release(&o2.base.dep_map, _THIS_IP_);
1842
1843         WWAI(&t);
1844         t2 = t;
1845         t2.stamp--;
1846
1847         ret = WWL(&o, &t);
1848         WARN_ON(ret);
1849
1850         ret = WWL(&o2, &t);
1851         WARN_ON(ret != -EDEADLK);
1852
1853         o2.ctx = NULL;
1854         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1855         mutex_unlock(&o2.base);
1856
1857         WWL(&o2, &t);
1858 }
1859
1860 static void ww_test_edeadlk_no_unlock_slow(void)
1861 {
1862         int ret;
1863
1864         mutex_lock(&o2.base);
1865         mutex_release(&o2.base.dep_map, _THIS_IP_);
1866         o2.ctx = &t2;
1867
1868         WWAI(&t);
1869         t2 = t;
1870         t2.stamp--;
1871
1872         ret = WWL(&o, &t);
1873         WARN_ON(ret);
1874
1875         ret = WWL(&o2, &t);
1876         WARN_ON(ret != -EDEADLK);
1877
1878         o2.ctx = NULL;
1879         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1880         mutex_unlock(&o2.base);
1881
1882         ww_mutex_lock_slow(&o2, &t);
1883 }
1884
1885 static void ww_test_edeadlk_acquire_more(void)
1886 {
1887         int ret;
1888
1889         mutex_lock(&o2.base);
1890         mutex_release(&o2.base.dep_map, _THIS_IP_);
1891         o2.ctx = &t2;
1892
1893         WWAI(&t);
1894         t2 = t;
1895         t2.stamp--;
1896
1897         ret = WWL(&o, &t);
1898         WARN_ON(ret);
1899
1900         ret = WWL(&o2, &t);
1901         WARN_ON(ret != -EDEADLK);
1902
1903         ret = WWL(&o3, &t);
1904 }
1905
1906 static void ww_test_edeadlk_acquire_more_slow(void)
1907 {
1908         int ret;
1909
1910         mutex_lock(&o2.base);
1911         mutex_release(&o2.base.dep_map, _THIS_IP_);
1912         o2.ctx = &t2;
1913
1914         WWAI(&t);
1915         t2 = t;
1916         t2.stamp--;
1917
1918         ret = WWL(&o, &t);
1919         WARN_ON(ret);
1920
1921         ret = WWL(&o2, &t);
1922         WARN_ON(ret != -EDEADLK);
1923
1924         ww_mutex_lock_slow(&o3, &t);
1925 }
1926
1927 static void ww_test_edeadlk_acquire_more_edeadlk(void)
1928 {
1929         int ret;
1930
1931         mutex_lock(&o2.base);
1932         mutex_release(&o2.base.dep_map, _THIS_IP_);
1933         o2.ctx = &t2;
1934
1935         mutex_lock(&o3.base);
1936         mutex_release(&o3.base.dep_map, _THIS_IP_);
1937         o3.ctx = &t2;
1938
1939         WWAI(&t);
1940         t2 = t;
1941         t2.stamp--;
1942
1943         ret = WWL(&o, &t);
1944         WARN_ON(ret);
1945
1946         ret = WWL(&o2, &t);
1947         WARN_ON(ret != -EDEADLK);
1948
1949         ret = WWL(&o3, &t);
1950         WARN_ON(ret != -EDEADLK);
1951 }
1952
1953 static void ww_test_edeadlk_acquire_more_edeadlk_slow(void)
1954 {
1955         int ret;
1956
1957         mutex_lock(&o2.base);
1958         mutex_release(&o2.base.dep_map, _THIS_IP_);
1959         o2.ctx = &t2;
1960
1961         mutex_lock(&o3.base);
1962         mutex_release(&o3.base.dep_map, _THIS_IP_);
1963         o3.ctx = &t2;
1964
1965         WWAI(&t);
1966         t2 = t;
1967         t2.stamp--;
1968
1969         ret = WWL(&o, &t);
1970         WARN_ON(ret);
1971
1972         ret = WWL(&o2, &t);
1973         WARN_ON(ret != -EDEADLK);
1974
1975         ww_mutex_lock_slow(&o3, &t);
1976 }
1977
1978 static void ww_test_edeadlk_acquire_wrong(void)
1979 {
1980         int ret;
1981
1982         mutex_lock(&o2.base);
1983         mutex_release(&o2.base.dep_map, _THIS_IP_);
1984         o2.ctx = &t2;
1985
1986         WWAI(&t);
1987         t2 = t;
1988         t2.stamp--;
1989
1990         ret = WWL(&o, &t);
1991         WARN_ON(ret);
1992
1993         ret = WWL(&o2, &t);
1994         WARN_ON(ret != -EDEADLK);
1995         if (!ret)
1996                 WWU(&o2);
1997
1998         WWU(&o);
1999
2000         ret = WWL(&o3, &t);
2001 }
2002
2003 static void ww_test_edeadlk_acquire_wrong_slow(void)
2004 {
2005         int ret;
2006
2007         mutex_lock(&o2.base);
2008         mutex_release(&o2.base.dep_map, _THIS_IP_);
2009         o2.ctx = &t2;
2010
2011         WWAI(&t);
2012         t2 = t;
2013         t2.stamp--;
2014
2015         ret = WWL(&o, &t);
2016         WARN_ON(ret);
2017
2018         ret = WWL(&o2, &t);
2019         WARN_ON(ret != -EDEADLK);
2020         if (!ret)
2021                 WWU(&o2);
2022
2023         WWU(&o);
2024
2025         ww_mutex_lock_slow(&o3, &t);
2026 }
2027
2028 static void ww_test_spin_nest_unlocked(void)
2029 {
2030         spin_lock_nest_lock(&lock_A, &o.base);
2031         U(A);
2032 }
2033
2034 /* This is not a deadlock, because we have X1 to serialize Y1 and Y2 */
2035 static void ww_test_spin_nest_lock(void)
2036 {
2037         spin_lock(&lock_X1);
2038         spin_lock_nest_lock(&lock_Y1, &lock_X1);
2039         spin_lock(&lock_A);
2040         spin_lock_nest_lock(&lock_Y2, &lock_X1);
2041         spin_unlock(&lock_A);
2042         spin_unlock(&lock_Y2);
2043         spin_unlock(&lock_Y1);
2044         spin_unlock(&lock_X1);
2045 }
2046
2047 static void ww_test_unneeded_slow(void)
2048 {
2049         WWAI(&t);
2050
2051         ww_mutex_lock_slow(&o, &t);
2052 }
2053
2054 static void ww_test_context_block(void)
2055 {
2056         int ret;
2057
2058         WWAI(&t);
2059
2060         ret = WWL(&o, &t);
2061         WARN_ON(ret);
2062         WWL1(&o2);
2063 }
2064
2065 static void ww_test_context_try(void)
2066 {
2067         int ret;
2068
2069         WWAI(&t);
2070
2071         ret = WWL(&o, &t);
2072         WARN_ON(ret);
2073
2074         ret = WWT(&o2);
2075         WARN_ON(!ret);
2076         WWU(&o2);
2077         WWU(&o);
2078 }
2079
2080 static void ww_test_context_context(void)
2081 {
2082         int ret;
2083
2084         WWAI(&t);
2085
2086         ret = WWL(&o, &t);
2087         WARN_ON(ret);
2088
2089         ret = WWL(&o2, &t);
2090         WARN_ON(ret);
2091
2092         WWU(&o2);
2093         WWU(&o);
2094 }
2095
2096 static void ww_test_try_block(void)
2097 {
2098         bool ret;
2099
2100         ret = WWT(&o);
2101         WARN_ON(!ret);
2102
2103         WWL1(&o2);
2104         WWU(&o2);
2105         WWU(&o);
2106 }
2107
2108 static void ww_test_try_try(void)
2109 {
2110         bool ret;
2111
2112         ret = WWT(&o);
2113         WARN_ON(!ret);
2114         ret = WWT(&o2);
2115         WARN_ON(!ret);
2116         WWU(&o2);
2117         WWU(&o);
2118 }
2119
2120 static void ww_test_try_context(void)
2121 {
2122         int ret;
2123
2124         ret = WWT(&o);
2125         WARN_ON(!ret);
2126
2127         WWAI(&t);
2128
2129         ret = WWL(&o2, &t);
2130         WARN_ON(ret);
2131 }
2132
2133 static void ww_test_block_block(void)
2134 {
2135         WWL1(&o);
2136         WWL1(&o2);
2137 }
2138
2139 static void ww_test_block_try(void)
2140 {
2141         bool ret;
2142
2143         WWL1(&o);
2144         ret = WWT(&o2);
2145         WARN_ON(!ret);
2146 }
2147
2148 static void ww_test_block_context(void)
2149 {
2150         int ret;
2151
2152         WWL1(&o);
2153         WWAI(&t);
2154
2155         ret = WWL(&o2, &t);
2156         WARN_ON(ret);
2157 }
2158
2159 static void ww_test_spin_block(void)
2160 {
2161         L(A);
2162         U(A);
2163
2164         WWL1(&o);
2165         L(A);
2166         U(A);
2167         WWU(&o);
2168
2169         L(A);
2170         WWL1(&o);
2171         WWU(&o);
2172         U(A);
2173 }
2174
2175 static void ww_test_spin_try(void)
2176 {
2177         bool ret;
2178
2179         L(A);
2180         U(A);
2181
2182         ret = WWT(&o);
2183         WARN_ON(!ret);
2184         L(A);
2185         U(A);
2186         WWU(&o);
2187
2188         L(A);
2189         ret = WWT(&o);
2190         WARN_ON(!ret);
2191         WWU(&o);
2192         U(A);
2193 }
2194
2195 static void ww_test_spin_context(void)
2196 {
2197         int ret;
2198
2199         L(A);
2200         U(A);
2201
2202         WWAI(&t);
2203
2204         ret = WWL(&o, &t);
2205         WARN_ON(ret);
2206         L(A);
2207         U(A);
2208         WWU(&o);
2209
2210         L(A);
2211         ret = WWL(&o, &t);
2212         WARN_ON(ret);
2213         WWU(&o);
2214         U(A);
2215 }
2216
2217 static void ww_tests(void)
2218 {
2219         printk("  --------------------------------------------------------------------------\n");
2220         printk("  | Wound/wait tests |\n");
2221         printk("  ---------------------\n");
2222
2223         print_testname("ww api failures");
2224         dotest(ww_test_fail_acquire, SUCCESS, LOCKTYPE_WW);
2225         dotest(ww_test_normal, SUCCESS, LOCKTYPE_WW);
2226         dotest(ww_test_unneeded_slow, FAILURE, LOCKTYPE_WW);
2227         pr_cont("\n");
2228
2229         print_testname("ww contexts mixing");
2230         dotest(ww_test_two_contexts, FAILURE, LOCKTYPE_WW);
2231         dotest(ww_test_diff_class, FAILURE, LOCKTYPE_WW);
2232         pr_cont("\n");
2233
2234         print_testname("finishing ww context");
2235         dotest(ww_test_context_done_twice, FAILURE, LOCKTYPE_WW);
2236         dotest(ww_test_context_unlock_twice, FAILURE, LOCKTYPE_WW);
2237         dotest(ww_test_context_fini_early, FAILURE, LOCKTYPE_WW);
2238         dotest(ww_test_context_lock_after_done, FAILURE, LOCKTYPE_WW);
2239         pr_cont("\n");
2240
2241         print_testname("locking mismatches");
2242         dotest(ww_test_object_unlock_twice, FAILURE, LOCKTYPE_WW);
2243         dotest(ww_test_object_lock_unbalanced, FAILURE, LOCKTYPE_WW);
2244         dotest(ww_test_object_lock_stale_context, FAILURE, LOCKTYPE_WW);
2245         pr_cont("\n");
2246
2247         print_testname("EDEADLK handling");
2248         dotest(ww_test_edeadlk_normal, SUCCESS, LOCKTYPE_WW);
2249         dotest(ww_test_edeadlk_normal_slow, SUCCESS, LOCKTYPE_WW);
2250         dotest(ww_test_edeadlk_no_unlock, FAILURE, LOCKTYPE_WW);
2251         dotest(ww_test_edeadlk_no_unlock_slow, FAILURE, LOCKTYPE_WW);
2252         dotest(ww_test_edeadlk_acquire_more, FAILURE, LOCKTYPE_WW);
2253         dotest(ww_test_edeadlk_acquire_more_slow, FAILURE, LOCKTYPE_WW);
2254         dotest(ww_test_edeadlk_acquire_more_edeadlk, FAILURE, LOCKTYPE_WW);
2255         dotest(ww_test_edeadlk_acquire_more_edeadlk_slow, FAILURE, LOCKTYPE_WW);
2256         dotest(ww_test_edeadlk_acquire_wrong, FAILURE, LOCKTYPE_WW);
2257         dotest(ww_test_edeadlk_acquire_wrong_slow, FAILURE, LOCKTYPE_WW);
2258         pr_cont("\n");
2259
2260         print_testname("spinlock nest unlocked");
2261         dotest(ww_test_spin_nest_unlocked, FAILURE, LOCKTYPE_WW);
2262         pr_cont("\n");
2263
2264         print_testname("spinlock nest test");
2265         dotest(ww_test_spin_nest_lock, SUCCESS, LOCKTYPE_WW);
2266         pr_cont("\n");
2267
2268         printk("  -----------------------------------------------------\n");
2269         printk("                                 |block | try  |context|\n");
2270         printk("  -----------------------------------------------------\n");
2271
2272         print_testname("context");
2273         dotest(ww_test_context_block, FAILURE, LOCKTYPE_WW);
2274         dotest(ww_test_context_try, SUCCESS, LOCKTYPE_WW);
2275         dotest(ww_test_context_context, SUCCESS, LOCKTYPE_WW);
2276         pr_cont("\n");
2277
2278         print_testname("try");
2279         dotest(ww_test_try_block, FAILURE, LOCKTYPE_WW);
2280         dotest(ww_test_try_try, SUCCESS, LOCKTYPE_WW);
2281         dotest(ww_test_try_context, FAILURE, LOCKTYPE_WW);
2282         pr_cont("\n");
2283
2284         print_testname("block");
2285         dotest(ww_test_block_block, FAILURE, LOCKTYPE_WW);
2286         dotest(ww_test_block_try, SUCCESS, LOCKTYPE_WW);
2287         dotest(ww_test_block_context, FAILURE, LOCKTYPE_WW);
2288         pr_cont("\n");
2289
2290         print_testname("spinlock");
2291         dotest(ww_test_spin_block, FAILURE, LOCKTYPE_WW);
2292         dotest(ww_test_spin_try, SUCCESS, LOCKTYPE_WW);
2293         dotest(ww_test_spin_context, FAILURE, LOCKTYPE_WW);
2294         pr_cont("\n");
2295 }
2296
2297
2298 /*
2299  * <in hardirq handler>
2300  * read_lock(&A);
2301  *                      <hardirq disable>
2302  *                      spin_lock(&B);
2303  * spin_lock(&B);
2304  *                      read_lock(&A);
2305  *
2306  * is a deadlock.
2307  */
2308 static void queued_read_lock_hardirq_RE_Er(void)
2309 {
2310         HARDIRQ_ENTER();
2311         read_lock(&rwlock_A);
2312         LOCK(B);
2313         UNLOCK(B);
2314         read_unlock(&rwlock_A);
2315         HARDIRQ_EXIT();
2316
2317         HARDIRQ_DISABLE();
2318         LOCK(B);
2319         read_lock(&rwlock_A);
2320         read_unlock(&rwlock_A);
2321         UNLOCK(B);
2322         HARDIRQ_ENABLE();
2323 }
2324
2325 /*
2326  * <in hardirq handler>
2327  * spin_lock(&B);
2328  *                      <hardirq disable>
2329  *                      read_lock(&A);
2330  * read_lock(&A);
2331  *                      spin_lock(&B);
2332  *
2333  * is not a deadlock.
2334  */
2335 static void queued_read_lock_hardirq_ER_rE(void)
2336 {
2337         HARDIRQ_ENTER();
2338         LOCK(B);
2339         read_lock(&rwlock_A);
2340         read_unlock(&rwlock_A);
2341         UNLOCK(B);
2342         HARDIRQ_EXIT();
2343
2344         HARDIRQ_DISABLE();
2345         read_lock(&rwlock_A);
2346         LOCK(B);
2347         UNLOCK(B);
2348         read_unlock(&rwlock_A);
2349         HARDIRQ_ENABLE();
2350 }
2351
2352 /*
2353  * <hardirq disable>
2354  * spin_lock(&B);
2355  *                      read_lock(&A);
2356  *                      <in hardirq handler>
2357  *                      spin_lock(&B);
2358  * read_lock(&A);
2359  *
2360  * is a deadlock. Because the two read_lock()s are both non-recursive readers.
2361  */
2362 static void queued_read_lock_hardirq_inversion(void)
2363 {
2364
2365         HARDIRQ_ENTER();
2366         LOCK(B);
2367         UNLOCK(B);
2368         HARDIRQ_EXIT();
2369
2370         HARDIRQ_DISABLE();
2371         LOCK(B);
2372         read_lock(&rwlock_A);
2373         read_unlock(&rwlock_A);
2374         UNLOCK(B);
2375         HARDIRQ_ENABLE();
2376
2377         read_lock(&rwlock_A);
2378         read_unlock(&rwlock_A);
2379 }
2380
2381 static void queued_read_lock_tests(void)
2382 {
2383         printk("  --------------------------------------------------------------------------\n");
2384         printk("  | queued read lock tests |\n");
2385         printk("  ---------------------------\n");
2386         print_testname("hardirq read-lock/lock-read");
2387         dotest(queued_read_lock_hardirq_RE_Er, FAILURE, LOCKTYPE_RWLOCK);
2388         pr_cont("\n");
2389
2390         print_testname("hardirq lock-read/read-lock");
2391         dotest(queued_read_lock_hardirq_ER_rE, SUCCESS, LOCKTYPE_RWLOCK);
2392         pr_cont("\n");
2393
2394         print_testname("hardirq inversion");
2395         dotest(queued_read_lock_hardirq_inversion, FAILURE, LOCKTYPE_RWLOCK);
2396         pr_cont("\n");
2397 }
2398
2399 static void fs_reclaim_correct_nesting(void)
2400 {
2401         fs_reclaim_acquire(GFP_KERNEL);
2402         might_alloc(GFP_NOFS);
2403         fs_reclaim_release(GFP_KERNEL);
2404 }
2405
2406 static void fs_reclaim_wrong_nesting(void)
2407 {
2408         fs_reclaim_acquire(GFP_KERNEL);
2409         might_alloc(GFP_KERNEL);
2410         fs_reclaim_release(GFP_KERNEL);
2411 }
2412
2413 static void fs_reclaim_protected_nesting(void)
2414 {
2415         unsigned int flags;
2416
2417         fs_reclaim_acquire(GFP_KERNEL);
2418         flags = memalloc_nofs_save();
2419         might_alloc(GFP_KERNEL);
2420         memalloc_nofs_restore(flags);
2421         fs_reclaim_release(GFP_KERNEL);
2422 }
2423
2424 static void fs_reclaim_tests(void)
2425 {
2426         printk("  --------------------\n");
2427         printk("  | fs_reclaim tests |\n");
2428         printk("  --------------------\n");
2429
2430         print_testname("correct nesting");
2431         dotest(fs_reclaim_correct_nesting, SUCCESS, 0);
2432         pr_cont("\n");
2433
2434         print_testname("wrong nesting");
2435         dotest(fs_reclaim_wrong_nesting, FAILURE, 0);
2436         pr_cont("\n");
2437
2438         print_testname("protected nesting");
2439         dotest(fs_reclaim_protected_nesting, SUCCESS, 0);
2440         pr_cont("\n");
2441 }
2442
2443 #define __guard(cleanup) __maybe_unused __attribute__((__cleanup__(cleanup)))
2444
2445 static void hardirq_exit(int *_)
2446 {
2447         HARDIRQ_EXIT();
2448 }
2449
2450 #define HARDIRQ_CONTEXT(name, ...)                                      \
2451         int hardirq_guard_##name __guard(hardirq_exit);                 \
2452         HARDIRQ_ENTER();
2453
2454 #define NOTTHREADED_HARDIRQ_CONTEXT(name, ...)                          \
2455         int notthreaded_hardirq_guard_##name __guard(hardirq_exit);     \
2456         local_irq_disable();                                            \
2457         __irq_enter();                                                  \
2458         WARN_ON(!in_irq());
2459
2460 static void softirq_exit(int *_)
2461 {
2462         SOFTIRQ_EXIT();
2463 }
2464
2465 #define SOFTIRQ_CONTEXT(name, ...)                              \
2466         int softirq_guard_##name __guard(softirq_exit);         \
2467         SOFTIRQ_ENTER();
2468
2469 static void rcu_exit(int *_)
2470 {
2471         rcu_read_unlock();
2472 }
2473
2474 #define RCU_CONTEXT(name, ...)                                  \
2475         int rcu_guard_##name __guard(rcu_exit);                 \
2476         rcu_read_lock();
2477
2478 static void rcu_bh_exit(int *_)
2479 {
2480         rcu_read_unlock_bh();
2481 }
2482
2483 #define RCU_BH_CONTEXT(name, ...)                               \
2484         int rcu_bh_guard_##name __guard(rcu_bh_exit);           \
2485         rcu_read_lock_bh();
2486
2487 static void rcu_sched_exit(int *_)
2488 {
2489         rcu_read_unlock_sched();
2490 }
2491
2492 #define RCU_SCHED_CONTEXT(name, ...)                            \
2493         int rcu_sched_guard_##name __guard(rcu_sched_exit);     \
2494         rcu_read_lock_sched();
2495
2496 static void rcu_callback_exit(int *_)
2497 {
2498         rcu_lock_release(&rcu_callback_map);
2499 }
2500
2501 #define RCU_CALLBACK_CONTEXT(name, ...)                                 \
2502         int rcu_callback_guard_##name __guard(rcu_callback_exit);       \
2503         rcu_lock_acquire(&rcu_callback_map);
2504
2505
2506 static void raw_spinlock_exit(raw_spinlock_t **lock)
2507 {
2508         raw_spin_unlock(*lock);
2509 }
2510
2511 #define RAW_SPINLOCK_CONTEXT(name, lock)                                                \
2512         raw_spinlock_t *raw_spinlock_guard_##name __guard(raw_spinlock_exit) = &(lock); \
2513         raw_spin_lock(&(lock));
2514
2515 static void spinlock_exit(spinlock_t **lock)
2516 {
2517         spin_unlock(*lock);
2518 }
2519
2520 #define SPINLOCK_CONTEXT(name, lock)                                            \
2521         spinlock_t *spinlock_guard_##name __guard(spinlock_exit) = &(lock);     \
2522         spin_lock(&(lock));
2523
2524 static void mutex_exit(struct mutex **lock)
2525 {
2526         mutex_unlock(*lock);
2527 }
2528
2529 #define MUTEX_CONTEXT(name, lock)                                       \
2530         struct mutex *mutex_guard_##name __guard(mutex_exit) = &(lock); \
2531         mutex_lock(&(lock));
2532
2533 #define GENERATE_2_CONTEXT_TESTCASE(outer, outer_lock, inner, inner_lock)       \
2534                                                                                 \
2535 static void __maybe_unused inner##_in_##outer(void)                             \
2536 {                                                                               \
2537         outer##_CONTEXT(_, outer_lock);                                         \
2538         {                                                                       \
2539                 inner##_CONTEXT(_, inner_lock);                                 \
2540         }                                                                       \
2541 }
2542
2543 /*
2544  * wait contexts (considering PREEMPT_RT)
2545  *
2546  * o: inner is allowed in outer
2547  * x: inner is disallowed in outer
2548  *
2549  *       \  inner |  RCU  | RAW_SPIN | SPIN | MUTEX
2550  * outer  \       |       |          |      |
2551  * ---------------+-------+----------+------+-------
2552  * HARDIRQ        |   o   |    o     |  o   |  x
2553  * ---------------+-------+----------+------+-------
2554  * NOTTHREADED_IRQ|   o   |    o     |  x   |  x
2555  * ---------------+-------+----------+------+-------
2556  * SOFTIRQ        |   o   |    o     |  o   |  x
2557  * ---------------+-------+----------+------+-------
2558  * RCU            |   o   |    o     |  o   |  x
2559  * ---------------+-------+----------+------+-------
2560  * RCU_BH         |   o   |    o     |  o   |  x
2561  * ---------------+-------+----------+------+-------
2562  * RCU_CALLBACK   |   o   |    o     |  o   |  x
2563  * ---------------+-------+----------+------+-------
2564  * RCU_SCHED      |   o   |    o     |  x   |  x
2565  * ---------------+-------+----------+------+-------
2566  * RAW_SPIN       |   o   |    o     |  x   |  x
2567  * ---------------+-------+----------+------+-------
2568  * SPIN           |   o   |    o     |  o   |  x
2569  * ---------------+-------+----------+------+-------
2570  * MUTEX          |   o   |    o     |  o   |  o
2571  * ---------------+-------+----------+------+-------
2572  */
2573
2574 #define GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(inner, inner_lock)            \
2575 GENERATE_2_CONTEXT_TESTCASE(HARDIRQ, , inner, inner_lock)                       \
2576 GENERATE_2_CONTEXT_TESTCASE(NOTTHREADED_HARDIRQ, , inner, inner_lock)           \
2577 GENERATE_2_CONTEXT_TESTCASE(SOFTIRQ, , inner, inner_lock)                       \
2578 GENERATE_2_CONTEXT_TESTCASE(RCU, , inner, inner_lock)                           \
2579 GENERATE_2_CONTEXT_TESTCASE(RCU_BH, , inner, inner_lock)                        \
2580 GENERATE_2_CONTEXT_TESTCASE(RCU_CALLBACK, , inner, inner_lock)                  \
2581 GENERATE_2_CONTEXT_TESTCASE(RCU_SCHED, , inner, inner_lock)                     \
2582 GENERATE_2_CONTEXT_TESTCASE(RAW_SPINLOCK, raw_lock_A, inner, inner_lock)        \
2583 GENERATE_2_CONTEXT_TESTCASE(SPINLOCK, lock_A, inner, inner_lock)                \
2584 GENERATE_2_CONTEXT_TESTCASE(MUTEX, mutex_A, inner, inner_lock)
2585
2586 GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(RCU, )
2587 GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(RAW_SPINLOCK, raw_lock_B)
2588 GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(SPINLOCK, lock_B)
2589 GENERATE_2_CONTEXT_TESTCASE_FOR_ALL_OUTER(MUTEX, mutex_B)
2590
2591 /* the outer context allows all kinds of preemption */
2592 #define DO_CONTEXT_TESTCASE_OUTER_PREEMPTIBLE(outer)                    \
2593         dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK);               \
2594         dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN);        \
2595         dotest(SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN);            \
2596         dotest(MUTEX_in_##outer, SUCCESS, LOCKTYPE_MUTEX);              \
2597
2598 /*
2599  * the outer context only allows the preemption introduced by spinlock_t (which
2600  * is a sleepable lock for PREEMPT_RT)
2601  */
2602 #define DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(outer)            \
2603         dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK);               \
2604         dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN);        \
2605         dotest(SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN);            \
2606         dotest(MUTEX_in_##outer, FAILURE, LOCKTYPE_MUTEX);              \
2607
2608 /* the outer doesn't allows any kind of preemption */
2609 #define DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(outer)                        \
2610         dotest(RCU_in_##outer, SUCCESS, LOCKTYPE_RWLOCK);               \
2611         dotest(RAW_SPINLOCK_in_##outer, SUCCESS, LOCKTYPE_SPIN);        \
2612         dotest(SPINLOCK_in_##outer, FAILURE, LOCKTYPE_SPIN);            \
2613         dotest(MUTEX_in_##outer, FAILURE, LOCKTYPE_MUTEX);              \
2614
2615 static void wait_context_tests(void)
2616 {
2617         printk("  --------------------------------------------------------------------------\n");
2618         printk("  | wait context tests |\n");
2619         printk("  --------------------------------------------------------------------------\n");
2620         printk("                                 | rcu  | raw  | spin |mutex |\n");
2621         printk("  --------------------------------------------------------------------------\n");
2622         print_testname("in hardirq context");
2623         DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(HARDIRQ);
2624         pr_cont("\n");
2625
2626         print_testname("in hardirq context (not threaded)");
2627         DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(NOTTHREADED_HARDIRQ);
2628         pr_cont("\n");
2629
2630         print_testname("in softirq context");
2631         DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(SOFTIRQ);
2632         pr_cont("\n");
2633
2634         print_testname("in RCU context");
2635         DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(RCU);
2636         pr_cont("\n");
2637
2638         print_testname("in RCU-bh context");
2639         DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(RCU_BH);
2640         pr_cont("\n");
2641
2642         print_testname("in RCU callback context");
2643         DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(RCU_CALLBACK);
2644         pr_cont("\n");
2645
2646         print_testname("in RCU-sched context");
2647         DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(RCU_SCHED);
2648         pr_cont("\n");
2649
2650         print_testname("in RAW_SPINLOCK context");
2651         DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(RAW_SPINLOCK);
2652         pr_cont("\n");
2653
2654         print_testname("in SPINLOCK context");
2655         DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(SPINLOCK);
2656         pr_cont("\n");
2657
2658         print_testname("in MUTEX context");
2659         DO_CONTEXT_TESTCASE_OUTER_PREEMPTIBLE(MUTEX);
2660         pr_cont("\n");
2661 }
2662
2663 static void local_lock_2(void)
2664 {
2665         local_lock_acquire(&local_A);   /* IRQ-ON */
2666         local_lock_release(&local_A);
2667
2668         HARDIRQ_ENTER();
2669         spin_lock(&lock_A);             /* IN-IRQ */
2670         spin_unlock(&lock_A);
2671         HARDIRQ_EXIT()
2672
2673         HARDIRQ_DISABLE();
2674         spin_lock(&lock_A);
2675         local_lock_acquire(&local_A);   /* IN-IRQ <-> IRQ-ON cycle, false */
2676         local_lock_release(&local_A);
2677         spin_unlock(&lock_A);
2678         HARDIRQ_ENABLE();
2679 }
2680
2681 static void local_lock_3A(void)
2682 {
2683         local_lock_acquire(&local_A);   /* IRQ-ON */
2684         spin_lock(&lock_B);             /* IRQ-ON */
2685         spin_unlock(&lock_B);
2686         local_lock_release(&local_A);
2687
2688         HARDIRQ_ENTER();
2689         spin_lock(&lock_A);             /* IN-IRQ */
2690         spin_unlock(&lock_A);
2691         HARDIRQ_EXIT()
2692
2693         HARDIRQ_DISABLE();
2694         spin_lock(&lock_A);
2695         local_lock_acquire(&local_A);   /* IN-IRQ <-> IRQ-ON cycle only if we count local_lock(), false */
2696         local_lock_release(&local_A);
2697         spin_unlock(&lock_A);
2698         HARDIRQ_ENABLE();
2699 }
2700
2701 static void local_lock_3B(void)
2702 {
2703         local_lock_acquire(&local_A);   /* IRQ-ON */
2704         spin_lock(&lock_B);             /* IRQ-ON */
2705         spin_unlock(&lock_B);
2706         local_lock_release(&local_A);
2707
2708         HARDIRQ_ENTER();
2709         spin_lock(&lock_A);             /* IN-IRQ */
2710         spin_unlock(&lock_A);
2711         HARDIRQ_EXIT()
2712
2713         HARDIRQ_DISABLE();
2714         spin_lock(&lock_A);
2715         local_lock_acquire(&local_A);   /* IN-IRQ <-> IRQ-ON cycle only if we count local_lock(), false */
2716         local_lock_release(&local_A);
2717         spin_unlock(&lock_A);
2718         HARDIRQ_ENABLE();
2719
2720         HARDIRQ_DISABLE();
2721         spin_lock(&lock_A);
2722         spin_lock(&lock_B);             /* IN-IRQ <-> IRQ-ON cycle, true */
2723         spin_unlock(&lock_B);
2724         spin_unlock(&lock_A);
2725         HARDIRQ_DISABLE();
2726
2727 }
2728
2729 static void local_lock_tests(void)
2730 {
2731         printk("  --------------------------------------------------------------------------\n");
2732         printk("  | local_lock tests |\n");
2733         printk("  ---------------------\n");
2734
2735         print_testname("local_lock inversion  2");
2736         dotest(local_lock_2, SUCCESS, LOCKTYPE_LL);
2737         pr_cont("\n");
2738
2739         print_testname("local_lock inversion 3A");
2740         dotest(local_lock_3A, SUCCESS, LOCKTYPE_LL);
2741         pr_cont("\n");
2742
2743         print_testname("local_lock inversion 3B");
2744         dotest(local_lock_3B, FAILURE, LOCKTYPE_LL);
2745         pr_cont("\n");
2746 }
2747
2748 void locking_selftest(void)
2749 {
2750         /*
2751          * Got a locking failure before the selftest ran?
2752          */
2753         if (!debug_locks) {
2754                 printk("----------------------------------\n");
2755                 printk("| Locking API testsuite disabled |\n");
2756                 printk("----------------------------------\n");
2757                 return;
2758         }
2759
2760         /*
2761          * treats read_lock() as recursive read locks for testing purpose
2762          */
2763         force_read_lock_recursive = 1;
2764
2765         /*
2766          * Run the testsuite:
2767          */
2768         printk("------------------------\n");
2769         printk("| Locking API testsuite:\n");
2770         printk("----------------------------------------------------------------------------\n");
2771         printk("                                 | spin |wlock |rlock |mutex | wsem | rsem |\n");
2772         printk("  --------------------------------------------------------------------------\n");
2773
2774         init_shared_classes();
2775         lockdep_set_selftest_task(current);
2776
2777         DO_TESTCASE_6R("A-A deadlock", AA);
2778         DO_TESTCASE_6R("A-B-B-A deadlock", ABBA);
2779         DO_TESTCASE_6R("A-B-B-C-C-A deadlock", ABBCCA);
2780         DO_TESTCASE_6R("A-B-C-A-B-C deadlock", ABCABC);
2781         DO_TESTCASE_6R("A-B-B-C-C-D-D-A deadlock", ABBCCDDA);
2782         DO_TESTCASE_6R("A-B-C-D-B-D-D-A deadlock", ABCDBDDA);
2783         DO_TESTCASE_6R("A-B-C-D-B-C-D-A deadlock", ABCDBCDA);
2784         DO_TESTCASE_6("double unlock", double_unlock);
2785         DO_TESTCASE_6("initialize held", init_held);
2786
2787         printk("  --------------------------------------------------------------------------\n");
2788         print_testname("recursive read-lock");
2789         pr_cont("             |");
2790         dotest(rlock_AA1, SUCCESS, LOCKTYPE_RWLOCK);
2791         pr_cont("             |");
2792         dotest(rsem_AA1, FAILURE, LOCKTYPE_RWSEM);
2793         pr_cont("\n");
2794
2795         print_testname("recursive read-lock #2");
2796         pr_cont("             |");
2797         dotest(rlock_AA1B, SUCCESS, LOCKTYPE_RWLOCK);
2798         pr_cont("             |");
2799         dotest(rsem_AA1B, FAILURE, LOCKTYPE_RWSEM);
2800         pr_cont("\n");
2801
2802         print_testname("mixed read-write-lock");
2803         pr_cont("             |");
2804         dotest(rlock_AA2, FAILURE, LOCKTYPE_RWLOCK);
2805         pr_cont("             |");
2806         dotest(rsem_AA2, FAILURE, LOCKTYPE_RWSEM);
2807         pr_cont("\n");
2808
2809         print_testname("mixed write-read-lock");
2810         pr_cont("             |");
2811         dotest(rlock_AA3, FAILURE, LOCKTYPE_RWLOCK);
2812         pr_cont("             |");
2813         dotest(rsem_AA3, FAILURE, LOCKTYPE_RWSEM);
2814         pr_cont("\n");
2815
2816         print_testname("mixed read-lock/lock-write ABBA");
2817         pr_cont("             |");
2818         dotest(rlock_ABBA1, FAILURE, LOCKTYPE_RWLOCK);
2819         pr_cont("             |");
2820         dotest(rwsem_ABBA1, FAILURE, LOCKTYPE_RWSEM);
2821
2822         print_testname("mixed read-lock/lock-read ABBA");
2823         pr_cont("             |");
2824         dotest(rlock_ABBA2, SUCCESS, LOCKTYPE_RWLOCK);
2825         pr_cont("             |");
2826         dotest(rwsem_ABBA2, FAILURE, LOCKTYPE_RWSEM);
2827
2828         print_testname("mixed write-lock/lock-write ABBA");
2829         pr_cont("             |");
2830         dotest(rlock_ABBA3, FAILURE, LOCKTYPE_RWLOCK);
2831         pr_cont("             |");
2832         dotest(rwsem_ABBA3, FAILURE, LOCKTYPE_RWSEM);
2833
2834         print_testname("chain cached mixed R-L/L-W ABBA");
2835         pr_cont("             |");
2836         dotest(rlock_chaincache_ABBA1, FAILURE, LOCKTYPE_RWLOCK);
2837
2838         DO_TESTCASE_6x1RRB("rlock W1R2/W2R3/W3R1", W1R2_W2R3_W3R1);
2839         DO_TESTCASE_6x1RRB("rlock W1W2/R2R3/W3R1", W1W2_R2R3_W3R1);
2840         DO_TESTCASE_6x1RR("rlock W1W2/R2R3/R3W1", W1W2_R2R3_R3W1);
2841         DO_TESTCASE_6x1RR("rlock W1R2/R2R3/W3W1", W1R2_R2R3_W3W1);
2842
2843         printk("  --------------------------------------------------------------------------\n");
2844
2845         /*
2846          * irq-context testcases:
2847          */
2848         DO_TESTCASE_2x6("irqs-on + irq-safe-A", irqsafe1);
2849         DO_TESTCASE_2x3("sirq-safe-A => hirqs-on", irqsafe2A);
2850         DO_TESTCASE_2x6("safe-A + irqs-on", irqsafe2B);
2851         DO_TESTCASE_6x6("safe-A + unsafe-B #1", irqsafe3);
2852         DO_TESTCASE_6x6("safe-A + unsafe-B #2", irqsafe4);
2853         DO_TESTCASE_6x6RW("irq lock-inversion", irq_inversion);
2854
2855         DO_TESTCASE_6x2x2RW("irq read-recursion", irq_read_recursion);
2856         DO_TESTCASE_6x2x2RW("irq read-recursion #2", irq_read_recursion2);
2857         DO_TESTCASE_6x2x2RW("irq read-recursion #3", irq_read_recursion3);
2858
2859         ww_tests();
2860
2861         force_read_lock_recursive = 0;
2862         /*
2863          * queued_read_lock() specific test cases can be put here
2864          */
2865         if (IS_ENABLED(CONFIG_QUEUED_RWLOCKS))
2866                 queued_read_lock_tests();
2867
2868         fs_reclaim_tests();
2869
2870         /* Wait context test cases that are specific for RAW_LOCK_NESTING */
2871         if (IS_ENABLED(CONFIG_PROVE_RAW_LOCK_NESTING))
2872                 wait_context_tests();
2873
2874         local_lock_tests();
2875
2876         if (unexpected_testcase_failures) {
2877                 printk("-----------------------------------------------------------------\n");
2878                 debug_locks = 0;
2879                 printk("BUG: %3d unexpected failures (out of %3d) - debugging disabled! |\n",
2880                         unexpected_testcase_failures, testcase_total);
2881                 printk("-----------------------------------------------------------------\n");
2882         } else if (expected_testcase_failures && testcase_successes) {
2883                 printk("--------------------------------------------------------\n");
2884                 printk("%3d out of %3d testcases failed, as expected. |\n",
2885                         expected_testcase_failures, testcase_total);
2886                 printk("----------------------------------------------------\n");
2887                 debug_locks = 1;
2888         } else if (expected_testcase_failures && !testcase_successes) {
2889                 printk("--------------------------------------------------------\n");
2890                 printk("All %3d testcases failed, as expected. |\n",
2891                         expected_testcase_failures);
2892                 printk("----------------------------------------\n");
2893                 debug_locks = 1;
2894         } else {
2895                 printk("-------------------------------------------------------\n");
2896                 printk("Good, all %3d testcases passed! |\n",
2897                         testcase_successes);
2898                 printk("---------------------------------\n");
2899                 debug_locks = 1;
2900         }
2901         lockdep_set_selftest_task(NULL);
2902         debug_locks_silent = 0;
2903 }