lib/debugobjects.c

   1 /*
   2  * Generic infrastructure for lifetime debugging of objects.
   3  *
   4  * Started by Thomas Gleixner
   5  *
   6  * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de>
   7  *
   8  * For licencing details see kernel-base/COPYING
   9  */
  10
  11 #define pr_fmt(fmt) "ODEBUG: " fmt
  12
  13 #include <linux/debugobjects.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/sched.h>
  16 #include <linux/sched/task_stack.h>
  17 #include <linux/seq_file.h>
  18 #include <linux/debugfs.h>
  19 #include <linux/slab.h>
  20 #include <linux/hash.h>
  21 #include <linux/kmemleak.h>
  22
  23 #define ODEBUG_HASH_BITS        14
  24 #define ODEBUG_HASH_SIZE        (1 << ODEBUG_HASH_BITS)
  25
  26 #define ODEBUG_POOL_SIZE        1024
  27 #define ODEBUG_POOL_MIN_LEVEL   256
  28 #define ODEBUG_POOL_PERCPU_SIZE 64
  29
  30 #define ODEBUG_CHUNK_SHIFT      PAGE_SHIFT
  31 #define ODEBUG_CHUNK_SIZE       (1 << ODEBUG_CHUNK_SHIFT)
  32 #define ODEBUG_CHUNK_MASK       (~(ODEBUG_CHUNK_SIZE - 1))
  33
  34 struct debug_bucket {
  35         struct hlist_head       list;
  36         raw_spinlock_t          lock;
  37 };
  38
  39 /*
  40  * Debug object percpu free list
  41  * Access is protected by disabling irq
  42  */
  43 struct debug_percpu_free {
  44         struct hlist_head       free_objs;
  45         int                     obj_free;
  46 };
  47
  48 static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
  49
  50 static struct debug_bucket      obj_hash[ODEBUG_HASH_SIZE];
  51
  52 static struct debug_obj         obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
  53
  54 static DEFINE_RAW_SPINLOCK(pool_lock);
  55
  56 static HLIST_HEAD(obj_pool);
  57 static HLIST_HEAD(obj_to_free);
  58
  59 /*
  60  * Because of the presence of percpu free pools, obj_pool_free will
  61  * under-count those in the percpu free pools. Similarly, obj_pool_used
  62  * will over-count those in the percpu free pools. Adjustments will be
  63  * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
  64  * can be off.
  65  */
  66 static int                      obj_pool_min_free = ODEBUG_POOL_SIZE;
  67 static int                      obj_pool_free = ODEBUG_POOL_SIZE;
  68 static int                      obj_pool_used;
  69 static int                      obj_pool_max_used;
  70 /* The number of objs on the global free list */
  71 static int                      obj_nr_tofree;
  72
  73 static int                      debug_objects_maxchain __read_mostly;
  74 static int __maybe_unused       debug_objects_maxchecked __read_mostly;
  75 static int                      debug_objects_fixups __read_mostly;
  76 static int                      debug_objects_warnings __read_mostly;
  77 static int                      debug_objects_enabled __read_mostly
  78                                 = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;
  79 static int                      debug_objects_pool_size __read_mostly
  80                                 = ODEBUG_POOL_SIZE;
  81 static int                      debug_objects_pool_min_level __read_mostly
  82                                 = ODEBUG_POOL_MIN_LEVEL;
  83 static struct debug_obj_descr   *descr_test  __read_mostly;
  84 static struct kmem_cache        *obj_cache __read_mostly;
  85
  86 /*
  87  * Track numbers of kmem_cache_alloc()/free() calls done.
  88  */
  89 static int                      debug_objects_allocated;
  90 static int                      debug_objects_freed;
  91
  92 static void free_obj_work(struct work_struct *work);
  93 static DECLARE_WORK(debug_obj_work, free_obj_work);
  94
  95 static int __init enable_object_debug(char *str)
  96 {
  97         debug_objects_enabled = 1;
  98         return 0;
  99 }
 100
 101 static int __init disable_object_debug(char *str)
 102 {
 103         debug_objects_enabled = 0;
 104         return 0;
 105 }
 106
 107 early_param("debug_objects", enable_object_debug);
 108 early_param("no_debug_objects", disable_object_debug);
 109
 110 static const char *obj_states[ODEBUG_STATE_MAX] = {
 111         [ODEBUG_STATE_NONE]             = "none",
 112         [ODEBUG_STATE_INIT]             = "initialized",
 113         [ODEBUG_STATE_INACTIVE]         = "inactive",
 114         [ODEBUG_STATE_ACTIVE]           = "active",
 115         [ODEBUG_STATE_DESTROYED]        = "destroyed",
 116         [ODEBUG_STATE_NOTAVAILABLE]     = "not available",
 117 };
 118
 119 static void fill_pool(void)
 120 {
 121         gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
 122         struct debug_obj *new, *obj;
 123         unsigned long flags;
 124
 125         if (likely(obj_pool_free >= debug_objects_pool_min_level))
 126                 return;
 127
 128         /*
 129          * Reuse objs from the global free list; they will be reinitialized
 130          * when allocating.
 131          */
 132         while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
 133                 raw_spin_lock_irqsave(&pool_lock, flags);
 134                 /*
 135                  * Recheck with the lock held as the worker thread might have
 136                  * won the race and freed the global free list already.
 137                  */
 138                 if (obj_nr_tofree) {
 139                         obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
 140                         hlist_del(&obj->node);
 141                         obj_nr_tofree--;
 142                         hlist_add_head(&obj->node, &obj_pool);
 143                         obj_pool_free++;
 144                 }
 145                 raw_spin_unlock_irqrestore(&pool_lock, flags);
 146         }
 147
 148         if (unlikely(!obj_cache))
 149                 return;
 150
 151         while (obj_pool_free < debug_objects_pool_min_level) {
 152
 153                 new = kmem_cache_zalloc(obj_cache, gfp);
 154                 if (!new)
 155                         return;
 156
 157                 raw_spin_lock_irqsave(&pool_lock, flags);
 158                 hlist_add_head(&new->node, &obj_pool);
 159                 debug_objects_allocated++;
 160                 obj_pool_free++;
 161                 raw_spin_unlock_irqrestore(&pool_lock, flags);
 162         }
 163 }
 164
 165 /*
 166  * Lookup an object in the hash bucket.
 167  */
 168 static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b)
 169 {
 170         struct debug_obj *obj;
 171         int cnt = 0;
 172
 173         hlist_for_each_entry(obj, &b->list, node) {
 174                 cnt++;
 175                 if (obj->object == addr)
 176                         return obj;
 177         }
 178         if (cnt > debug_objects_maxchain)
 179                 debug_objects_maxchain = cnt;
 180
 181         return NULL;
 182 }
 183
 184 /*
 185  * Allocate a new object from the hlist
 186  */
 187 static struct debug_obj *__alloc_object(struct hlist_head *list)
 188 {
 189         struct debug_obj *obj = NULL;
 190
 191         if (list->first) {
 192                 obj = hlist_entry(list->first, typeof(*obj), node);
 193                 hlist_del(&obj->node);
 194         }
 195
 196         return obj;
 197 }
 198
 199 static struct debug_obj *
 200 alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
 201 {
 202         struct debug_percpu_free *percpu_pool;
 203         struct debug_obj *obj;
 204
 205         if (likely(obj_cache)) {
 206                 percpu_pool = this_cpu_ptr(&percpu_obj_pool);
 207                 obj = __alloc_object(&percpu_pool->free_objs);
 208                 if (obj) {
 209                         percpu_pool->obj_free--;
 210                         goto init_obj;
 211                 }
 212         }
 213
 214         raw_spin_lock(&pool_lock);
 215         obj = __alloc_object(&obj_pool);
 216         if (obj) {
 217                 obj_pool_used++;
 218                 if (obj_pool_used > obj_pool_max_used)
 219                         obj_pool_max_used = obj_pool_used;
 220
 221                 obj_pool_free--;
 222                 if (obj_pool_free < obj_pool_min_free)
 223                         obj_pool_min_free = obj_pool_free;
 224         }
 225         raw_spin_unlock(&pool_lock);
 226
 227 init_obj:
 228         if (obj) {
 229                 obj->object = addr;
 230                 obj->descr  = descr;
 231                 obj->state  = ODEBUG_STATE_NONE;
 232                 obj->astate = 0;
 233                 hlist_add_head(&obj->node, &b->list);
 234         }
 235         return obj;
 236 }
 237
 238 /*
 239  * workqueue function to free objects.
 240  *
 241  * To reduce contention on the global pool_lock, the actual freeing of
 242  * debug objects will be delayed if the pool_lock is busy.
 243  */
 244 static void free_obj_work(struct work_struct *work)
 245 {
 246         struct hlist_node *tmp;
 247         struct debug_obj *obj;
 248         unsigned long flags;
 249         HLIST_HEAD(tofree);
 250
 251         if (!raw_spin_trylock_irqsave(&pool_lock, flags))
 252                 return;
 253
 254         /*
 255          * The objs on the pool list might be allocated before the work is
 256          * run, so recheck if pool list it full or not, if not fill pool
 257          * list from the global free list
 258          */
 259         while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
 260                 obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
 261                 hlist_del(&obj->node);
 262                 hlist_add_head(&obj->node, &obj_pool);
 263                 obj_pool_free++;
 264                 obj_nr_tofree--;
 265         }
 266
 267         /*
 268          * Pool list is already full and there are still objs on the free
 269          * list. Move remaining free objs to a temporary list to free the
 270          * memory outside the pool_lock held region.
 271          */
 272         if (obj_nr_tofree) {
 273                 hlist_move_list(&obj_to_free, &tofree);
 274                 debug_objects_freed += obj_nr_tofree;
 275                 obj_nr_tofree = 0;
 276         }
 277         raw_spin_unlock_irqrestore(&pool_lock, flags);
 278
 279         hlist_for_each_entry_safe(obj, tmp, &tofree, node) {
 280                 hlist_del(&obj->node);
 281                 kmem_cache_free(obj_cache, obj);
 282         }
 283 }
 284
 285 static bool __free_object(struct debug_obj *obj)
 286 {
 287         unsigned long flags;
 288         bool work;
 289         struct debug_percpu_free *percpu_pool;
 290
 291         local_irq_save(flags);
 292         /*
 293          * Try to free it into the percpu pool first.
 294          */
 295         percpu_pool = this_cpu_ptr(&percpu_obj_pool);
 296         if (obj_cache && percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
 297                 hlist_add_head(&obj->node, &percpu_pool->free_objs);
 298                 percpu_pool->obj_free++;
 299                 local_irq_restore(flags);
 300                 return false;
 301         }
 302
 303         raw_spin_lock(&pool_lock);
 304         work = (obj_pool_free > debug_objects_pool_size) && obj_cache;
 305         obj_pool_used--;
 306
 307         if (work) {
 308                 obj_nr_tofree++;
 309                 hlist_add_head(&obj->node, &obj_to_free);
 310         } else {
 311                 obj_pool_free++;
 312                 hlist_add_head(&obj->node, &obj_pool);
 313         }
 314         raw_spin_unlock(&pool_lock);
 315         local_irq_restore(flags);
 316         return work;
 317 }
 318
 319 /*
 320  * Put the object back into the pool and schedule work to free objects
 321  * if necessary.
 322  */
 323 static void free_object(struct debug_obj *obj)
 324 {
 325         if (__free_object(obj))
 326                 schedule_work(&debug_obj_work);
 327 }
 328
 329 /*
 330  * We run out of memory. That means we probably have tons of objects
 331  * allocated.
 332  */
 333 static void debug_objects_oom(void)
 334 {
 335         struct debug_bucket *db = obj_hash;
 336         struct hlist_node *tmp;
 337         HLIST_HEAD(freelist);
 338         struct debug_obj *obj;
 339         unsigned long flags;
 340         int i;
 341
 342         pr_warn("Out of memory. ODEBUG disabled\n");
 343
 344         for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
 345                 raw_spin_lock_irqsave(&db->lock, flags);
 346                 hlist_move_list(&db->list, &freelist);
 347                 raw_spin_unlock_irqrestore(&db->lock, flags);
 348
 349                 /* Now free them */
 350                 hlist_for_each_entry_safe(obj, tmp, &freelist, node) {
 351                         hlist_del(&obj->node);
 352                         free_object(obj);
 353                 }
 354         }
 355 }
 356
 357 /*
 358  * We use the pfn of the address for the hash. That way we can check
 359  * for freed objects simply by checking the affected bucket.
 360  */
 361 static struct debug_bucket *get_bucket(unsigned long addr)
 362 {
 363         unsigned long hash;
 364
 365         hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS);
 366         return &obj_hash[hash];
 367 }
 368
 369 static void debug_print_object(struct debug_obj *obj, char *msg)
 370 {
 371         struct debug_obj_descr *descr = obj->descr;
 372         static int limit;
 373
 374         /*
 375          * Don't report if lookup_object_or_alloc() by the current thread
 376          * failed because lookup_object_or_alloc()/debug_objects_oom() by a
 377          * concurrent thread turned off debug_objects_enabled and cleared
 378          * the hash buckets.
 379          */
 380         if (!debug_objects_enabled)
 381                 return;
 382
 383         if (limit < 5 && descr != descr_test) {
 384                 void *hint = descr->debug_hint ?
 385                         descr->debug_hint(obj->object) : NULL;
 386                 limit++;
 387                 WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "
 388                                  "object type: %s hint: %pS\n",
 389                         msg, obj_states[obj->state], obj->astate,
 390                         descr->name, hint);
 391         }
 392         debug_objects_warnings++;
 393 }
 394
 395 /*
 396  * Try to repair the damage, so we have a better chance to get useful
 397  * debug output.
 398  */
 399 static bool
 400 debug_object_fixup(bool (*fixup)(void *addr, enum debug_obj_state state),
 401                    void * addr, enum debug_obj_state state)
 402 {
 403         if (fixup && fixup(addr, state)) {
 404                 debug_objects_fixups++;
 405                 return true;
 406         }
 407         return false;
 408 }
 409
 410 static void debug_object_is_on_stack(void *addr, int onstack)
 411 {
 412         int is_on_stack;
 413         static int limit;
 414
 415         if (limit > 4)
 416                 return;
 417
 418         is_on_stack = object_is_on_stack(addr);
 419         if (is_on_stack == onstack)
 420                 return;
 421
 422         limit++;
 423         if (is_on_stack)
 424                 pr_warn("object %p is on stack %p, but NOT annotated.\n", addr,
 425                          task_stack_page(current));
 426         else
 427                 pr_warn("object %p is NOT on stack %p, but annotated.\n", addr,
 428                          task_stack_page(current));
 429
 430         WARN_ON(1);
 431 }
 432
 433 static struct debug_obj *lookup_object_or_alloc(void *addr, struct debug_bucket *b,
 434                                                 struct debug_obj_descr *descr,
 435                                                 bool onstack, bool alloc_ifstatic)
 436 {
 437         struct debug_obj *obj = lookup_object(addr, b);
 438         enum debug_obj_state state = ODEBUG_STATE_NONE;
 439
 440         if (likely(obj))
 441                 return obj;
 442
 443         /*
 444          * debug_object_init() unconditionally allocates untracked
 445          * objects. It does not matter whether it is a static object or
 446          * not.
 447          *
 448          * debug_object_assert_init() and debug_object_activate() allow
 449          * allocation only if the descriptor callback confirms that the
 450          * object is static and considered initialized. For non-static
 451          * objects the allocation needs to be done from the fixup callback.
 452          */
 453         if (unlikely(alloc_ifstatic)) {
 454                 if (!descr->is_static_object || !descr->is_static_object(addr))
 455                         return ERR_PTR(-ENOENT);
 456                 /* Statically allocated objects are considered initialized */
 457                 state = ODEBUG_STATE_INIT;
 458         }
 459
 460         obj = alloc_object(addr, b, descr);
 461         if (likely(obj)) {
 462                 obj->state = state;
 463                 debug_object_is_on_stack(addr, onstack);
 464                 return obj;
 465         }
 466
 467         /* Out of memory. Do the cleanup outside of the locked region */
 468         debug_objects_enabled = 0;
 469         return NULL;
 470 }
 471
 472 static void debug_objects_fill_pool(void)
 473 {
 474         /*
 475          * On RT enabled kernels the pool refill must happen in preemptible
 476          * context:
 477          */
 478         if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible())
 479                 fill_pool();
 480 }
 481
 482 static void
 483 __debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
 484 {
 485         enum debug_obj_state state;
 486         struct debug_bucket *db;
 487         struct debug_obj *obj;
 488         unsigned long flags;
 489
 490         debug_objects_fill_pool();
 491
 492         db = get_bucket((unsigned long) addr);
 493
 494         raw_spin_lock_irqsave(&db->lock, flags);
 495
 496         obj = lookup_object_or_alloc(addr, db, descr, onstack, false);
 497         if (unlikely(!obj)) {
 498                 raw_spin_unlock_irqrestore(&db->lock, flags);
 499                 debug_objects_oom();
 500                 return;
 501         }
 502
 503         switch (obj->state) {
 504         case ODEBUG_STATE_NONE:
 505         case ODEBUG_STATE_INIT:
 506         case ODEBUG_STATE_INACTIVE:
 507                 obj->state = ODEBUG_STATE_INIT;
 508                 break;
 509
 510         case ODEBUG_STATE_ACTIVE:
 511                 state = obj->state;
 512                 raw_spin_unlock_irqrestore(&db->lock, flags);
 513                 debug_print_object(obj, "init");
 514                 debug_object_fixup(descr->fixup_init, addr, state);
 515                 return;
 516
 517         case ODEBUG_STATE_DESTROYED:
 518                 raw_spin_unlock_irqrestore(&db->lock, flags);
 519                 debug_print_object(obj, "init");
 520                 return;
 521         default:
 522                 break;
 523         }
 524
 525         raw_spin_unlock_irqrestore(&db->lock, flags);
 526 }
 527
 528 /**
 529  * debug_object_init - debug checks when an object is initialized
 530  * @addr:       address of the object
 531  * @descr:      pointer to an object specific debug description structure
 532  */
 533 void debug_object_init(void *addr, struct debug_obj_descr *descr)
 534 {
 535         if (!debug_objects_enabled)
 536                 return;
 537
 538         __debug_object_init(addr, descr, 0);
 539 }
 540 EXPORT_SYMBOL_GPL(debug_object_init);
 541
 542 /**
 543  * debug_object_init_on_stack - debug checks when an object on stack is
 544  *                              initialized
 545  * @addr:       address of the object
 546  * @descr:      pointer to an object specific debug description structure
 547  */
 548 void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
 549 {
 550         if (!debug_objects_enabled)
 551                 return;
 552
 553         __debug_object_init(addr, descr, 1);
 554 }
 555 EXPORT_SYMBOL_GPL(debug_object_init_on_stack);
 556
 557 /**
 558  * debug_object_activate - debug checks when an object is activated
 559  * @addr:       address of the object
 560  * @descr:      pointer to an object specific debug description structure
 561  * Returns 0 for success, -EINVAL for check failed.
 562  */
 563 int debug_object_activate(void *addr, struct debug_obj_descr *descr)
 564 {
 565         struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr };
 566         enum debug_obj_state state;
 567         struct debug_bucket *db;
 568         struct debug_obj *obj;
 569         unsigned long flags;
 570         int ret;
 571
 572         if (!debug_objects_enabled)
 573                 return 0;
 574
 575         debug_objects_fill_pool();
 576
 577         db = get_bucket((unsigned long) addr);
 578
 579         raw_spin_lock_irqsave(&db->lock, flags);
 580
 581         obj = lookup_object_or_alloc(addr, db, descr, false, true);
 582         if (likely(!IS_ERR_OR_NULL(obj))) {
 583                 bool print_object = false;
 584
 585                 switch (obj->state) {
 586                 case ODEBUG_STATE_INIT:
 587                 case ODEBUG_STATE_INACTIVE:
 588                         obj->state = ODEBUG_STATE_ACTIVE;
 589                         ret = 0;
 590                         break;
 591
 592                 case ODEBUG_STATE_ACTIVE:
 593                         state = obj->state;
 594                         raw_spin_unlock_irqrestore(&db->lock, flags);
 595                         debug_print_object(obj, "activate");
 596                         ret = debug_object_fixup(descr->fixup_activate, addr, state);
 597                         return ret ? 0 : -EINVAL;
 598
 599                 case ODEBUG_STATE_DESTROYED:
 600                         print_object = true;
 601                         ret = -EINVAL;
 602                         break;
 603                 default:
 604                         ret = 0;
 605                         break;
 606                 }
 607                 raw_spin_unlock_irqrestore(&db->lock, flags);
 608                 if (print_object)
 609                         debug_print_object(obj, "activate");
 610                 return ret;
 611         }
 612
 613         raw_spin_unlock_irqrestore(&db->lock, flags);
 614
 615         /* If NULL the allocation has hit OOM */
 616         if (!obj) {
 617                 debug_objects_oom();
 618                 return 0;
 619         }
 620
 621         /* Object is neither static nor tracked. It's not initialized */
 622         debug_print_object(&o, "activate");
 623         ret = debug_object_fixup(descr->fixup_activate, addr, ODEBUG_STATE_NOTAVAILABLE);
 624         return ret ? 0 : -EINVAL;
 625 }
 626 EXPORT_SYMBOL_GPL(debug_object_activate);
 627
 628 /**
 629  * debug_object_deactivate - debug checks when an object is deactivated
 630  * @addr:       address of the object
 631  * @descr:      pointer to an object specific debug description structure
 632  */
 633 void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
 634 {
 635         struct debug_bucket *db;
 636         struct debug_obj *obj;
 637         unsigned long flags;
 638         bool print_object = false;
 639
 640         if (!debug_objects_enabled)
 641                 return;
 642
 643         db = get_bucket((unsigned long) addr);
 644
 645         raw_spin_lock_irqsave(&db->lock, flags);
 646
 647         obj = lookup_object(addr, db);
 648         if (obj) {
 649                 switch (obj->state) {
 650                 case ODEBUG_STATE_INIT:
 651                 case ODEBUG_STATE_INACTIVE:
 652                 case ODEBUG_STATE_ACTIVE:
 653                         if (!obj->astate)
 654                                 obj->state = ODEBUG_STATE_INACTIVE;
 655                         else
 656                                 print_object = true;
 657                         break;
 658
 659                 case ODEBUG_STATE_DESTROYED:
 660                         print_object = true;
 661                         break;
 662                 default:
 663                         break;
 664                 }
 665         }
 666
 667         raw_spin_unlock_irqrestore(&db->lock, flags);
 668         if (!obj) {
 669                 struct debug_obj o = { .object = addr,
 670                                        .state = ODEBUG_STATE_NOTAVAILABLE,
 671                                        .descr = descr };
 672
 673                 debug_print_object(&o, "deactivate");
 674         } else if (print_object) {
 675                 debug_print_object(obj, "deactivate");
 676         }
 677 }
 678 EXPORT_SYMBOL_GPL(debug_object_deactivate);
 679
 680 /**
 681  * debug_object_destroy - debug checks when an object is destroyed
 682  * @addr:       address of the object
 683  * @descr:      pointer to an object specific debug description structure
 684  */
 685 void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
 686 {
 687         enum debug_obj_state state;
 688         struct debug_bucket *db;
 689         struct debug_obj *obj;
 690         unsigned long flags;
 691         bool print_object = false;
 692
 693         if (!debug_objects_enabled)
 694                 return;
 695
 696         db = get_bucket((unsigned long) addr);
 697
 698         raw_spin_lock_irqsave(&db->lock, flags);
 699
 700         obj = lookup_object(addr, db);
 701         if (!obj)
 702                 goto out_unlock;
 703
 704         switch (obj->state) {
 705         case ODEBUG_STATE_NONE:
 706         case ODEBUG_STATE_INIT:
 707         case ODEBUG_STATE_INACTIVE:
 708                 obj->state = ODEBUG_STATE_DESTROYED;
 709                 break;
 710         case ODEBUG_STATE_ACTIVE:
 711                 state = obj->state;
 712                 raw_spin_unlock_irqrestore(&db->lock, flags);
 713                 debug_print_object(obj, "destroy");
 714                 debug_object_fixup(descr->fixup_destroy, addr, state);
 715                 return;
 716
 717         case ODEBUG_STATE_DESTROYED:
 718                 print_object = true;
 719                 break;
 720         default:
 721                 break;
 722         }
 723 out_unlock:
 724         raw_spin_unlock_irqrestore(&db->lock, flags);
 725         if (print_object)
 726                 debug_print_object(obj, "destroy");
 727 }
 728 EXPORT_SYMBOL_GPL(debug_object_destroy);
 729
 730 /**
 731  * debug_object_free - debug checks when an object is freed
 732  * @addr:       address of the object
 733  * @descr:      pointer to an object specific debug description structure
 734  */
 735 void debug_object_free(void *addr, struct debug_obj_descr *descr)
 736 {
 737         enum debug_obj_state state;
 738         struct debug_bucket *db;
 739         struct debug_obj *obj;
 740         unsigned long flags;
 741
 742         if (!debug_objects_enabled)
 743                 return;
 744
 745         db = get_bucket((unsigned long) addr);
 746
 747         raw_spin_lock_irqsave(&db->lock, flags);
 748
 749         obj = lookup_object(addr, db);
 750         if (!obj)
 751                 goto out_unlock;
 752
 753         switch (obj->state) {
 754         case ODEBUG_STATE_ACTIVE:
 755                 state = obj->state;
 756                 raw_spin_unlock_irqrestore(&db->lock, flags);
 757                 debug_print_object(obj, "free");
 758                 debug_object_fixup(descr->fixup_free, addr, state);
 759                 return;
 760         default:
 761                 hlist_del(&obj->node);
 762                 raw_spin_unlock_irqrestore(&db->lock, flags);
 763                 free_object(obj);
 764                 return;
 765         }
 766 out_unlock:
 767         raw_spin_unlock_irqrestore(&db->lock, flags);
 768 }
 769 EXPORT_SYMBOL_GPL(debug_object_free);
 770
 771 /**
 772  * debug_object_assert_init - debug checks when object should be init-ed
 773  * @addr:       address of the object
 774  * @descr:      pointer to an object specific debug description structure
 775  */
 776 void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
 777 {
 778         struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr };
 779         struct debug_bucket *db;
 780         struct debug_obj *obj;
 781         unsigned long flags;
 782
 783         if (!debug_objects_enabled)
 784                 return;
 785
 786         debug_objects_fill_pool();
 787
 788         db = get_bucket((unsigned long) addr);
 789
 790         raw_spin_lock_irqsave(&db->lock, flags);
 791         obj = lookup_object_or_alloc(addr, db, descr, false, true);
 792         raw_spin_unlock_irqrestore(&db->lock, flags);
 793         if (likely(!IS_ERR_OR_NULL(obj)))
 794                 return;
 795
 796         /* If NULL the allocation has hit OOM */
 797         if (!obj) {
 798                 debug_objects_oom();
 799                 return;
 800         }
 801
 802         /* Object is neither tracked nor static. It's not initialized. */
 803         debug_print_object(&o, "assert_init");
 804         debug_object_fixup(descr->fixup_assert_init, addr, ODEBUG_STATE_NOTAVAILABLE);
 805 }
 806 EXPORT_SYMBOL_GPL(debug_object_assert_init);
 807
 808 /**
 809  * debug_object_active_state - debug checks object usage state machine
 810  * @addr:       address of the object
 811  * @descr:      pointer to an object specific debug description structure
 812  * @expect:     expected state
 813  * @next:       state to move to if expected state is found
 814  */
 815 void
 816 debug_object_active_state(void *addr, struct debug_obj_descr *descr,
 817                           unsigned int expect, unsigned int next)
 818 {
 819         struct debug_bucket *db;
 820         struct debug_obj *obj;
 821         unsigned long flags;
 822         bool print_object = false;
 823
 824         if (!debug_objects_enabled)
 825                 return;
 826
 827         db = get_bucket((unsigned long) addr);
 828
 829         raw_spin_lock_irqsave(&db->lock, flags);
 830
 831         obj = lookup_object(addr, db);
 832         if (obj) {
 833                 switch (obj->state) {
 834                 case ODEBUG_STATE_ACTIVE:
 835                         if (obj->astate == expect)
 836                                 obj->astate = next;
 837                         else
 838                                 print_object = true;
 839                         break;
 840
 841                 default:
 842                         print_object = true;
 843                         break;
 844                 }
 845         }
 846
 847         raw_spin_unlock_irqrestore(&db->lock, flags);
 848         if (!obj) {
 849                 struct debug_obj o = { .object = addr,
 850                                        .state = ODEBUG_STATE_NOTAVAILABLE,
 851                                        .descr = descr };
 852
 853                 debug_print_object(&o, "active_state");
 854         } else if (print_object) {
 855                 debug_print_object(obj, "active_state");
 856         }
 857 }
 858 EXPORT_SYMBOL_GPL(debug_object_active_state);
 859
 860 #ifdef CONFIG_DEBUG_OBJECTS_FREE
 861 static void __debug_check_no_obj_freed(const void *address, unsigned long size)
 862 {
 863         unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
 864         struct debug_obj_descr *descr;
 865         enum debug_obj_state state;
 866         struct debug_bucket *db;
 867         struct hlist_node *tmp;
 868         struct debug_obj *obj;
 869         int cnt, objs_checked = 0;
 870         bool work = false;
 871
 872         saddr = (unsigned long) address;
 873         eaddr = saddr + size;
 874         paddr = saddr & ODEBUG_CHUNK_MASK;
 875         chunks = ((eaddr - paddr) + (ODEBUG_CHUNK_SIZE - 1));
 876         chunks >>= ODEBUG_CHUNK_SHIFT;
 877
 878         for (;chunks > 0; chunks--, paddr += ODEBUG_CHUNK_SIZE) {
 879                 db = get_bucket(paddr);
 880
 881 repeat:
 882                 cnt = 0;
 883                 raw_spin_lock_irqsave(&db->lock, flags);
 884                 hlist_for_each_entry_safe(obj, tmp, &db->list, node) {
 885                         cnt++;
 886                         oaddr = (unsigned long) obj->object;
 887                         if (oaddr < saddr || oaddr >= eaddr)
 888                                 continue;
 889
 890                         switch (obj->state) {
 891                         case ODEBUG_STATE_ACTIVE:
 892                                 descr = obj->descr;
 893                                 state = obj->state;
 894                                 raw_spin_unlock_irqrestore(&db->lock, flags);
 895                                 debug_print_object(obj, "free");
 896                                 debug_object_fixup(descr->fixup_free,
 897                                                    (void *) oaddr, state);
 898                                 goto repeat;
 899                         default:
 900                                 hlist_del(&obj->node);
 901                                 work |= __free_object(obj);
 902                                 break;
 903                         }
 904                 }
 905                 raw_spin_unlock_irqrestore(&db->lock, flags);
 906
 907                 if (cnt > debug_objects_maxchain)
 908                         debug_objects_maxchain = cnt;
 909
 910                 objs_checked += cnt;
 911         }
 912
 913         if (objs_checked > debug_objects_maxchecked)
 914                 debug_objects_maxchecked = objs_checked;
 915
 916         /* Schedule work to actually kmem_cache_free() objects */
 917         if (work)
 918                 schedule_work(&debug_obj_work);
 919 }
 920
 921 void debug_check_no_obj_freed(const void *address, unsigned long size)
 922 {
 923         if (debug_objects_enabled)
 924                 __debug_check_no_obj_freed(address, size);
 925 }
 926 #endif
 927
 928 #ifdef CONFIG_DEBUG_FS
 929
 930 static int debug_stats_show(struct seq_file *m, void *v)
 931 {
 932         int cpu, obj_percpu_free = 0;
 933
 934         for_each_possible_cpu(cpu)
 935                 obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
 936
 937         seq_printf(m, "max_chain     :%d\n", debug_objects_maxchain);
 938         seq_printf(m, "max_checked   :%d\n", debug_objects_maxchecked);
 939         seq_printf(m, "warnings      :%d\n", debug_objects_warnings);
 940         seq_printf(m, "fixups        :%d\n", debug_objects_fixups);
 941         seq_printf(m, "pool_free     :%d\n", obj_pool_free + obj_percpu_free);
 942         seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
 943         seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
 944         seq_printf(m, "pool_used     :%d\n", obj_pool_used - obj_percpu_free);
 945         seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
 946         seq_printf(m, "on_free_list  :%d\n", obj_nr_tofree);
 947         seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
 948         seq_printf(m, "objs_freed    :%d\n", debug_objects_freed);
 949         return 0;
 950 }
 951
 952 static int debug_stats_open(struct inode *inode, struct file *filp)
 953 {
 954         return single_open(filp, debug_stats_show, NULL);
 955 }
 956
 957 static const struct file_operations debug_stats_fops = {
 958         .open           = debug_stats_open,
 959         .read           = seq_read,
 960         .llseek         = seq_lseek,
 961         .release        = single_release,
 962 };
 963
 964 static int __init debug_objects_init_debugfs(void)
 965 {
 966         struct dentry *dbgdir, *dbgstats;
 967
 968         if (!debug_objects_enabled)
 969                 return 0;
 970
 971         dbgdir = debugfs_create_dir("debug_objects", NULL);
 972         if (!dbgdir)
 973                 return -ENOMEM;
 974
 975         dbgstats = debugfs_create_file("stats", 0444, dbgdir, NULL,
 976                                        &debug_stats_fops);
 977         if (!dbgstats)
 978                 goto err;
 979
 980         return 0;
 981
 982 err:
 983         debugfs_remove(dbgdir);
 984
 985         return -ENOMEM;
 986 }
 987 __initcall(debug_objects_init_debugfs);
 988
 989 #else
 990 static inline void debug_objects_init_debugfs(void) { }
 991 #endif
 992
 993 #ifdef CONFIG_DEBUG_OBJECTS_SELFTEST
 994
 995 /* Random data structure for the self test */
 996 struct self_test {
 997         unsigned long   dummy1[6];
 998         int             static_init;
 999         unsigned long   dummy2[3];
1000 };
1001
1002 static __initdata struct debug_obj_descr descr_type_test;
1003
1004 static bool __init is_static_object(void *addr)
1005 {
1006         struct self_test *obj = addr;
1007
1008         return obj->static_init;
1009 }
1010
1011 /*
1012  * fixup_init is called when:
1013  * - an active object is initialized
1014  */
1015 static bool __init fixup_init(void *addr, enum debug_obj_state state)
1016 {
1017         struct self_test *obj = addr;
1018
1019         switch (state) {
1020         case ODEBUG_STATE_ACTIVE:
1021                 debug_object_deactivate(obj, &descr_type_test);
1022                 debug_object_init(obj, &descr_type_test);
1023                 return true;
1024         default:
1025                 return false;
1026         }
1027 }
1028
1029 /*
1030  * fixup_activate is called when:
1031  * - an active object is activated
1032  * - an unknown non-static object is activated
1033  */
1034 static bool __init fixup_activate(void *addr, enum debug_obj_state state)
1035 {
1036         struct self_test *obj = addr;
1037
1038         switch (state) {
1039         case ODEBUG_STATE_NOTAVAILABLE:
1040                 return true;
1041         case ODEBUG_STATE_ACTIVE:
1042                 debug_object_deactivate(obj, &descr_type_test);
1043                 debug_object_activate(obj, &descr_type_test);
1044                 return true;
1045
1046         default:
1047                 return false;
1048         }
1049 }
1050
1051 /*
1052  * fixup_destroy is called when:
1053  * - an active object is destroyed
1054  */
1055 static bool __init fixup_destroy(void *addr, enum debug_obj_state state)
1056 {
1057         struct self_test *obj = addr;
1058
1059         switch (state) {
1060         case ODEBUG_STATE_ACTIVE:
1061                 debug_object_deactivate(obj, &descr_type_test);
1062                 debug_object_destroy(obj, &descr_type_test);
1063                 return true;
1064         default:
1065                 return false;
1066         }
1067 }
1068
1069 /*
1070  * fixup_free is called when:
1071  * - an active object is freed
1072  */
1073 static bool __init fixup_free(void *addr, enum debug_obj_state state)
1074 {
1075         struct self_test *obj = addr;
1076
1077         switch (state) {
1078         case ODEBUG_STATE_ACTIVE:
1079                 debug_object_deactivate(obj, &descr_type_test);
1080                 debug_object_free(obj, &descr_type_test);
1081                 return true;
1082         default:
1083                 return false;
1084         }
1085 }
1086
1087 static int __init
1088 check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
1089 {
1090         struct debug_bucket *db;
1091         struct debug_obj *obj;
1092         unsigned long flags;
1093         int res = -EINVAL;
1094
1095         db = get_bucket((unsigned long) addr);
1096
1097         raw_spin_lock_irqsave(&db->lock, flags);
1098
1099         obj = lookup_object(addr, db);
1100         if (!obj && state != ODEBUG_STATE_NONE) {
1101                 WARN(1, KERN_ERR "ODEBUG: selftest object not found\n");
1102                 goto out;
1103         }
1104         if (obj && obj->state != state) {
1105                 WARN(1, KERN_ERR "ODEBUG: selftest wrong state: %d != %d\n",
1106                        obj->state, state);
1107                 goto out;
1108         }
1109         if (fixups != debug_objects_fixups) {
1110                 WARN(1, KERN_ERR "ODEBUG: selftest fixups failed %d != %d\n",
1111                        fixups, debug_objects_fixups);
1112                 goto out;
1113         }
1114         if (warnings != debug_objects_warnings) {
1115                 WARN(1, KERN_ERR "ODEBUG: selftest warnings failed %d != %d\n",
1116                        warnings, debug_objects_warnings);
1117                 goto out;
1118         }
1119         res = 0;
1120 out:
1121         raw_spin_unlock_irqrestore(&db->lock, flags);
1122         if (res)
1123                 debug_objects_enabled = 0;
1124         return res;
1125 }
1126
1127 static __initdata struct debug_obj_descr descr_type_test = {
1128         .name                   = "selftest",
1129         .is_static_object       = is_static_object,
1130         .fixup_init             = fixup_init,
1131         .fixup_activate         = fixup_activate,
1132         .fixup_destroy          = fixup_destroy,
1133         .fixup_free             = fixup_free,
1134 };
1135
1136 static __initdata struct self_test obj = { .static_init = 0 };
1137
1138 static void __init debug_objects_selftest(void)
1139 {
1140         int fixups, oldfixups, warnings, oldwarnings;
1141         unsigned long flags;
1142
1143         local_irq_save(flags);
1144
1145         fixups = oldfixups = debug_objects_fixups;
1146         warnings = oldwarnings = debug_objects_warnings;
1147         descr_test = &descr_type_test;
1148
1149         debug_object_init(&obj, &descr_type_test);
1150         if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1151                 goto out;
1152         debug_object_activate(&obj, &descr_type_test);
1153         if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1154                 goto out;
1155         debug_object_activate(&obj, &descr_type_test);
1156         if (check_results(&obj, ODEBUG_STATE_ACTIVE, ++fixups, ++warnings))
1157                 goto out;
1158         debug_object_deactivate(&obj, &descr_type_test);
1159         if (check_results(&obj, ODEBUG_STATE_INACTIVE, fixups, warnings))
1160                 goto out;
1161         debug_object_destroy(&obj, &descr_type_test);
1162         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, warnings))
1163                 goto out;
1164         debug_object_init(&obj, &descr_type_test);
1165         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1166                 goto out;
1167         debug_object_activate(&obj, &descr_type_test);
1168         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1169                 goto out;
1170         debug_object_deactivate(&obj, &descr_type_test);
1171         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1172                 goto out;
1173         debug_object_free(&obj, &descr_type_test);
1174         if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1175                 goto out;
1176
1177         obj.static_init = 1;
1178         debug_object_activate(&obj, &descr_type_test);
1179         if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1180                 goto out;
1181         debug_object_init(&obj, &descr_type_test);
1182         if (check_results(&obj, ODEBUG_STATE_INIT, ++fixups, ++warnings))
1183                 goto out;
1184         debug_object_free(&obj, &descr_type_test);
1185         if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1186                 goto out;
1187
1188 #ifdef CONFIG_DEBUG_OBJECTS_FREE
1189         debug_object_init(&obj, &descr_type_test);
1190         if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1191                 goto out;
1192         debug_object_activate(&obj, &descr_type_test);
1193         if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1194                 goto out;
1195         __debug_check_no_obj_freed(&obj, sizeof(obj));
1196         if (check_results(&obj, ODEBUG_STATE_NONE, ++fixups, ++warnings))
1197                 goto out;
1198 #endif
1199         pr_info("selftest passed\n");
1200
1201 out:
1202         debug_objects_fixups = oldfixups;
1203         debug_objects_warnings = oldwarnings;
1204         descr_test = NULL;
1205
1206         local_irq_restore(flags);
1207 }
1208 #else
1209 static inline void debug_objects_selftest(void) { }
1210 #endif
1211
1212 /*
1213  * Called during early boot to initialize the hash buckets and link
1214  * the static object pool objects into the poll list. After this call
1215  * the object tracker is fully operational.
1216  */
1217 void __init debug_objects_early_init(void)
1218 {
1219         int i;
1220
1221         for (i = 0; i < ODEBUG_HASH_SIZE; i++)
1222                 raw_spin_lock_init(&obj_hash[i].lock);
1223
1224         for (i = 0; i < ODEBUG_POOL_SIZE; i++)
1225                 hlist_add_head(&obj_static_pool[i].node, &obj_pool);
1226 }
1227
1228 /*
1229  * Convert the statically allocated objects to dynamic ones:
1230  */
1231 static int __init debug_objects_replace_static_objects(void)
1232 {
1233         struct debug_bucket *db = obj_hash;
1234         struct hlist_node *tmp;
1235         struct debug_obj *obj, *new;
1236         HLIST_HEAD(objects);
1237         int i, cnt = 0;
1238
1239         for (i = 0; i < ODEBUG_POOL_SIZE; i++) {
1240                 obj = kmem_cache_zalloc(obj_cache, GFP_KERNEL);
1241                 if (!obj)
1242                         goto free;
1243                 hlist_add_head(&obj->node, &objects);
1244         }
1245
1246         /*
1247          * When debug_objects_mem_init() is called we know that only
1248          * one CPU is up, so disabling interrupts is enough
1249          * protection. This avoids the lockdep hell of lock ordering.
1250          */
1251         local_irq_disable();
1252
1253         /* Remove the statically allocated objects from the pool */
1254         hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
1255                 hlist_del(&obj->node);
1256         /* Move the allocated objects to the pool */
1257         hlist_move_list(&objects, &obj_pool);
1258
1259         /* Replace the active object references */
1260         for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
1261                 hlist_move_list(&db->list, &objects);
1262
1263                 hlist_for_each_entry(obj, &objects, node) {
1264                         new = hlist_entry(obj_pool.first, typeof(*obj), node);
1265                         hlist_del(&new->node);
1266                         /* copy object data */
1267                         *new = *obj;
1268                         hlist_add_head(&new->node, &db->list);
1269                         cnt++;
1270                 }
1271         }
1272         local_irq_enable();
1273
1274         pr_debug("%d of %d active objects replaced\n",
1275                  cnt, obj_pool_used);
1276         return 0;
1277 free:
1278         hlist_for_each_entry_safe(obj, tmp, &objects, node) {
1279                 hlist_del(&obj->node);
1280                 kmem_cache_free(obj_cache, obj);
1281         }
1282         return -ENOMEM;
1283 }
1284
1285 /*
1286  * Called after the kmem_caches are functional to setup a dedicated
1287  * cache pool, which has the SLAB_DEBUG_OBJECTS flag set. This flag
1288  * prevents that the debug code is called on kmem_cache_free() for the
1289  * debug tracker objects to avoid recursive calls.
1290  */
1291 void __init debug_objects_mem_init(void)
1292 {
1293         int cpu;
1294
1295         if (!debug_objects_enabled)
1296                 return;
1297
1298         /*
1299          * Initialize the percpu object pools
1300          *
1301          * Initialization is not strictly necessary, but was done for
1302          * completeness.
1303          */
1304         for_each_possible_cpu(cpu)
1305                 INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
1306
1307         obj_cache = kmem_cache_create("debug_objects_cache",
1308                                       sizeof (struct debug_obj), 0,
1309                                       SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,
1310                                       NULL);
1311
1312         if (!obj_cache || debug_objects_replace_static_objects()) {
1313                 debug_objects_enabled = 0;
1314                 kmem_cache_destroy(obj_cache);
1315                 pr_warn("out of memory.\n");
1316         } else
1317                 debug_objects_selftest();
1318 }