fs/fscache/cookie.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* netfs cookie management
   3  *
   4  * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  *
   7  * See Documentation/filesystems/caching/netfs-api.rst for more information on
   8  * the netfs API.
   9  */
  10
  11 #define FSCACHE_DEBUG_LEVEL COOKIE
  12 #include <linux/module.h>
  13 #include <linux/slab.h>
  14 #include "internal.h"
  15
  16 struct kmem_cache *fscache_cookie_jar;
  17
  18 static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
  19
  20 #define fscache_cookie_hash_shift 15
  21 static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
  22
  23 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
  24                                             loff_t object_size);
  25 static int fscache_alloc_object(struct fscache_cache *cache,
  26                                 struct fscache_cookie *cookie);
  27 static int fscache_attach_object(struct fscache_cookie *cookie,
  28                                  struct fscache_object *object);
  29
  30 static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
  31 {
  32         struct hlist_node *object;
  33         const u8 *k;
  34         unsigned loop;
  35
  36         pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n",
  37                prefix, cookie, cookie->parent, cookie->flags,
  38                atomic_read(&cookie->n_children),
  39                atomic_read(&cookie->n_active));
  40         pr_err("%c-cookie d=%p n=%p\n",
  41                prefix, cookie->def, cookie->netfs_data);
  42
  43         object = READ_ONCE(cookie->backing_objects.first);
  44         if (object)
  45                 pr_err("%c-cookie o=%p\n",
  46                        prefix, hlist_entry(object, struct fscache_object, cookie_link));
  47
  48         pr_err("%c-key=[%u] '", prefix, cookie->key_len);
  49         k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
  50                 cookie->inline_key : cookie->key;
  51         for (loop = 0; loop < cookie->key_len; loop++)
  52                 pr_cont("%02x", k[loop]);
  53         pr_cont("'\n");
  54 }
  55
  56 void fscache_free_cookie(struct fscache_cookie *cookie)
  57 {
  58         if (cookie) {
  59                 BUG_ON(!hlist_empty(&cookie->backing_objects));
  60                 if (cookie->aux_len > sizeof(cookie->inline_aux))
  61                         kfree(cookie->aux);
  62                 if (cookie->key_len > sizeof(cookie->inline_key))
  63                         kfree(cookie->key);
  64                 kmem_cache_free(fscache_cookie_jar, cookie);
  65         }
  66 }
  67
  68 /*
  69  * Set the index key in a cookie.  The cookie struct has space for a 16-byte
  70  * key plus length and hash, but if that's not big enough, it's instead a
  71  * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
  72  * the key data.
  73  */
  74 static int fscache_set_key(struct fscache_cookie *cookie,
  75                            const void *index_key, size_t index_key_len)
  76 {
  77         u32 *buf;
  78         int bufs;
  79
  80         bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf));
  81
  82         if (index_key_len > sizeof(cookie->inline_key)) {
  83                 buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL);
  84                 if (!buf)
  85                         return -ENOMEM;
  86                 cookie->key = buf;
  87         } else {
  88                 buf = (u32 *)cookie->inline_key;
  89         }
  90
  91         memcpy(buf, index_key, index_key_len);
  92         cookie->key_hash = fscache_hash(0, buf, bufs);
  93         return 0;
  94 }
  95
  96 static long fscache_compare_cookie(const struct fscache_cookie *a,
  97                                    const struct fscache_cookie *b)
  98 {
  99         const void *ka, *kb;
 100
 101         if (a->key_hash != b->key_hash)
 102                 return (long)a->key_hash - (long)b->key_hash;
 103         if (a->parent != b->parent)
 104                 return (long)a->parent - (long)b->parent;
 105         if (a->key_len != b->key_len)
 106                 return (long)a->key_len - (long)b->key_len;
 107         if (a->type != b->type)
 108                 return (long)a->type - (long)b->type;
 109
 110         if (a->key_len <= sizeof(a->inline_key)) {
 111                 ka = &a->inline_key;
 112                 kb = &b->inline_key;
 113         } else {
 114                 ka = a->key;
 115                 kb = b->key;
 116         }
 117         return memcmp(ka, kb, a->key_len);
 118 }
 119
 120 /*
 121  * Allocate a cookie.
 122  */
 123 struct fscache_cookie *fscache_alloc_cookie(
 124         struct fscache_cookie *parent,
 125         const struct fscache_cookie_def *def,
 126         const void *index_key, size_t index_key_len,
 127         const void *aux_data, size_t aux_data_len,
 128         void *netfs_data,
 129         loff_t object_size)
 130 {
 131         struct fscache_cookie *cookie;
 132
 133         /* allocate and initialise a cookie */
 134         cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
 135         if (!cookie)
 136                 return NULL;
 137
 138         cookie->key_len = index_key_len;
 139         cookie->aux_len = aux_data_len;
 140
 141         if (fscache_set_key(cookie, index_key, index_key_len) < 0)
 142                 goto nomem;
 143
 144         if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
 145                 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
 146         } else {
 147                 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
 148                 if (!cookie->aux)
 149                         goto nomem;
 150         }
 151
 152         atomic_set(&cookie->usage, 1);
 153         atomic_set(&cookie->n_children, 0);
 154
 155         /* We keep the active count elevated until relinquishment to prevent an
 156          * attempt to wake up every time the object operations queue quiesces.
 157          */
 158         atomic_set(&cookie->n_active, 1);
 159
 160         cookie->def             = def;
 161         cookie->parent          = parent;
 162         cookie->netfs_data      = netfs_data;
 163         cookie->flags           = (1 << FSCACHE_COOKIE_NO_DATA_YET);
 164         cookie->type            = def->type;
 165         spin_lock_init(&cookie->lock);
 166         spin_lock_init(&cookie->stores_lock);
 167         INIT_HLIST_HEAD(&cookie->backing_objects);
 168
 169         /* radix tree insertion won't use the preallocation pool unless it's
 170          * told it may not wait */
 171         INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
 172         return cookie;
 173
 174 nomem:
 175         fscache_free_cookie(cookie);
 176         return NULL;
 177 }
 178
 179 /*
 180  * Attempt to insert the new cookie into the hash.  If there's a collision, we
 181  * return the old cookie if it's not in use and an error otherwise.
 182  */
 183 struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
 184 {
 185         struct fscache_cookie *cursor;
 186         struct hlist_bl_head *h;
 187         struct hlist_bl_node *p;
 188         unsigned int bucket;
 189
 190         bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 191         h = &fscache_cookie_hash[bucket];
 192
 193         hlist_bl_lock(h);
 194         hlist_bl_for_each_entry(cursor, p, h, hash_link) {
 195                 if (fscache_compare_cookie(candidate, cursor) == 0)
 196                         goto collision;
 197         }
 198
 199         __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
 200         fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
 201         atomic_inc(&candidate->parent->n_children);
 202         hlist_bl_add_head(&candidate->hash_link, h);
 203         hlist_bl_unlock(h);
 204         return candidate;
 205
 206 collision:
 207         if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
 208                 trace_fscache_cookie(cursor, fscache_cookie_collision,
 209                                      atomic_read(&cursor->usage));
 210                 pr_err("Duplicate cookie detected\n");
 211                 fscache_print_cookie(cursor, 'O');
 212                 fscache_print_cookie(candidate, 'N');
 213                 hlist_bl_unlock(h);
 214                 return NULL;
 215         }
 216
 217         fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
 218         hlist_bl_unlock(h);
 219         return cursor;
 220 }
 221
 222 /*
 223  * request a cookie to represent an object (index, datafile, xattr, etc)
 224  * - parent specifies the parent object
 225  *   - the top level index cookie for each netfs is stored in the fscache_netfs
 226  *     struct upon registration
 227  * - def points to the definition
 228  * - the netfs_data will be passed to the functions pointed to in *def
 229  * - all attached caches will be searched to see if they contain this object
 230  * - index objects aren't stored on disk until there's a dependent file that
 231  *   needs storing
 232  * - other objects are stored in a selected cache immediately, and all the
 233  *   indices forming the path to it are instantiated if necessary
 234  * - we never let on to the netfs about errors
 235  *   - we may set a negative cookie pointer, but that's okay
 236  */
 237 struct fscache_cookie *__fscache_acquire_cookie(
 238         struct fscache_cookie *parent,
 239         const struct fscache_cookie_def *def,
 240         const void *index_key, size_t index_key_len,
 241         const void *aux_data, size_t aux_data_len,
 242         void *netfs_data,
 243         loff_t object_size,
 244         bool enable)
 245 {
 246         struct fscache_cookie *candidate, *cookie;
 247
 248         BUG_ON(!def);
 249
 250         _enter("{%s},{%s},%p,%u",
 251                parent ? (char *) parent->def->name : "<no-parent>",
 252                def->name, netfs_data, enable);
 253
 254         if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
 255                 return NULL;
 256         if (!aux_data || !aux_data_len) {
 257                 aux_data = NULL;
 258                 aux_data_len = 0;
 259         }
 260
 261         fscache_stat(&fscache_n_acquires);
 262
 263         /* if there's no parent cookie, then we don't create one here either */
 264         if (!parent) {
 265                 fscache_stat(&fscache_n_acquires_null);
 266                 _leave(" [no parent]");
 267                 return NULL;
 268         }
 269
 270         /* validate the definition */
 271         BUG_ON(!def->name[0]);
 272
 273         BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
 274                parent->type != FSCACHE_COOKIE_TYPE_INDEX);
 275
 276         candidate = fscache_alloc_cookie(parent, def,
 277                                          index_key, index_key_len,
 278                                          aux_data, aux_data_len,
 279                                          netfs_data, object_size);
 280         if (!candidate) {
 281                 fscache_stat(&fscache_n_acquires_oom);
 282                 _leave(" [ENOMEM]");
 283                 return NULL;
 284         }
 285
 286         cookie = fscache_hash_cookie(candidate);
 287         if (!cookie) {
 288                 trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
 289                 goto out;
 290         }
 291
 292         if (cookie == candidate)
 293                 candidate = NULL;
 294
 295         switch (cookie->type) {
 296         case FSCACHE_COOKIE_TYPE_INDEX:
 297                 fscache_stat(&fscache_n_cookie_index);
 298                 break;
 299         case FSCACHE_COOKIE_TYPE_DATAFILE:
 300                 fscache_stat(&fscache_n_cookie_data);
 301                 break;
 302         default:
 303                 fscache_stat(&fscache_n_cookie_special);
 304                 break;
 305         }
 306
 307         trace_fscache_acquire(cookie);
 308
 309         if (enable) {
 310                 /* if the object is an index then we need do nothing more here
 311                  * - we create indices on disk when we need them as an index
 312                  * may exist in multiple caches */
 313                 if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 314                         if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {
 315                                 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 316                         } else {
 317                                 atomic_dec(&parent->n_children);
 318                                 fscache_cookie_put(cookie,
 319                                                    fscache_cookie_put_acquire_nobufs);
 320                                 fscache_stat(&fscache_n_acquires_nobufs);
 321                                 _leave(" = NULL");
 322                                 return NULL;
 323                         }
 324                 } else {
 325                         set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 326                 }
 327         }
 328
 329         fscache_stat(&fscache_n_acquires_ok);
 330
 331 out:
 332         fscache_free_cookie(candidate);
 333         return cookie;
 334 }
 335 EXPORT_SYMBOL(__fscache_acquire_cookie);
 336
 337 /*
 338  * Enable a cookie to permit it to accept new operations.
 339  */
 340 void __fscache_enable_cookie(struct fscache_cookie *cookie,
 341                              const void *aux_data,
 342                              loff_t object_size,
 343                              bool (*can_enable)(void *data),
 344                              void *data)
 345 {
 346         _enter("%p", cookie);
 347
 348         trace_fscache_enable(cookie);
 349
 350         wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 351                          TASK_UNINTERRUPTIBLE);
 352
 353         fscache_update_aux(cookie, aux_data);
 354
 355         if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 356                 goto out_unlock;
 357
 358         if (can_enable && !can_enable(data)) {
 359                 /* The netfs decided it didn't want to enable after all */
 360         } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 361                 /* Wait for outstanding disablement to complete */
 362                 __fscache_wait_on_invalidate(cookie);
 363
 364                 if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)
 365                         set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 366         } else {
 367                 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 368         }
 369
 370 out_unlock:
 371         clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 372         wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 373 }
 374 EXPORT_SYMBOL(__fscache_enable_cookie);
 375
 376 /*
 377  * acquire a non-index cookie
 378  * - this must make sure the index chain is instantiated and instantiate the
 379  *   object representation too
 380  */
 381 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
 382                                             loff_t object_size)
 383 {
 384         struct fscache_object *object;
 385         struct fscache_cache *cache;
 386         int ret;
 387
 388         _enter("");
 389
 390         set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
 391
 392         /* now we need to see whether the backing objects for this cookie yet
 393          * exist, if not there'll be nothing to search */
 394         down_read(&fscache_addremove_sem);
 395
 396         if (list_empty(&fscache_cache_list)) {
 397                 up_read(&fscache_addremove_sem);
 398                 _leave(" = 0 [no caches]");
 399                 return 0;
 400         }
 401
 402         /* select a cache in which to store the object */
 403         cache = fscache_select_cache_for_object(cookie->parent);
 404         if (!cache) {
 405                 up_read(&fscache_addremove_sem);
 406                 fscache_stat(&fscache_n_acquires_no_cache);
 407                 _leave(" = -ENOMEDIUM [no cache]");
 408                 return -ENOMEDIUM;
 409         }
 410
 411         _debug("cache %s", cache->tag->name);
 412
 413         set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
 414
 415         /* ask the cache to allocate objects for this cookie and its parent
 416          * chain */
 417         ret = fscache_alloc_object(cache, cookie);
 418         if (ret < 0) {
 419                 up_read(&fscache_addremove_sem);
 420                 _leave(" = %d", ret);
 421                 return ret;
 422         }
 423
 424         spin_lock(&cookie->lock);
 425         if (hlist_empty(&cookie->backing_objects)) {
 426                 spin_unlock(&cookie->lock);
 427                 goto unavailable;
 428         }
 429
 430         object = hlist_entry(cookie->backing_objects.first,
 431                              struct fscache_object, cookie_link);
 432
 433         fscache_set_store_limit(object, object_size);
 434
 435         /* initiate the process of looking up all the objects in the chain
 436          * (done by fscache_initialise_object()) */
 437         fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD);
 438
 439         spin_unlock(&cookie->lock);
 440
 441         /* we may be required to wait for lookup to complete at this point */
 442         if (!fscache_defer_lookup) {
 443                 _debug("non-deferred lookup %p", &cookie->flags);
 444                 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 445                             TASK_UNINTERRUPTIBLE);
 446                 _debug("complete");
 447                 if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
 448                         goto unavailable;
 449         }
 450
 451         up_read(&fscache_addremove_sem);
 452         _leave(" = 0 [deferred]");
 453         return 0;
 454
 455 unavailable:
 456         up_read(&fscache_addremove_sem);
 457         _leave(" = -ENOBUFS");
 458         return -ENOBUFS;
 459 }
 460
 461 /*
 462  * recursively allocate cache object records for a cookie/cache combination
 463  * - caller must be holding the addremove sem
 464  */
 465 static int fscache_alloc_object(struct fscache_cache *cache,
 466                                 struct fscache_cookie *cookie)
 467 {
 468         struct fscache_object *object;
 469         int ret;
 470
 471         _enter("%p,%p{%s}", cache, cookie, cookie->def->name);
 472
 473         spin_lock(&cookie->lock);
 474         hlist_for_each_entry(object, &cookie->backing_objects,
 475                              cookie_link) {
 476                 if (object->cache == cache)
 477                         goto object_already_extant;
 478         }
 479         spin_unlock(&cookie->lock);
 480
 481         /* ask the cache to allocate an object (we may end up with duplicate
 482          * objects at this stage, but we sort that out later) */
 483         fscache_stat(&fscache_n_cop_alloc_object);
 484         object = cache->ops->alloc_object(cache, cookie);
 485         fscache_stat_d(&fscache_n_cop_alloc_object);
 486         if (IS_ERR(object)) {
 487                 fscache_stat(&fscache_n_object_no_alloc);
 488                 ret = PTR_ERR(object);
 489                 goto error;
 490         }
 491
 492         ASSERTCMP(object->cookie, ==, cookie);
 493         fscache_stat(&fscache_n_object_alloc);
 494
 495         object->debug_id = atomic_inc_return(&fscache_object_debug_id);
 496
 497         _debug("ALLOC OBJ%x: %s {%lx}",
 498                object->debug_id, cookie->def->name, object->events);
 499
 500         ret = fscache_alloc_object(cache, cookie->parent);
 501         if (ret < 0)
 502                 goto error_put;
 503
 504         /* only attach if we managed to allocate all we needed, otherwise
 505          * discard the object we just allocated and instead use the one
 506          * attached to the cookie */
 507         if (fscache_attach_object(cookie, object) < 0) {
 508                 fscache_stat(&fscache_n_cop_put_object);
 509                 cache->ops->put_object(object, fscache_obj_put_attach_fail);
 510                 fscache_stat_d(&fscache_n_cop_put_object);
 511         }
 512
 513         _leave(" = 0");
 514         return 0;
 515
 516 object_already_extant:
 517         ret = -ENOBUFS;
 518         if (fscache_object_is_dying(object) ||
 519             fscache_cache_is_broken(object)) {
 520                 spin_unlock(&cookie->lock);
 521                 goto error;
 522         }
 523         spin_unlock(&cookie->lock);
 524         _leave(" = 0 [found]");
 525         return 0;
 526
 527 error_put:
 528         fscache_stat(&fscache_n_cop_put_object);
 529         cache->ops->put_object(object, fscache_obj_put_alloc_fail);
 530         fscache_stat_d(&fscache_n_cop_put_object);
 531 error:
 532         _leave(" = %d", ret);
 533         return ret;
 534 }
 535
 536 /*
 537  * attach a cache object to a cookie
 538  */
 539 static int fscache_attach_object(struct fscache_cookie *cookie,
 540                                  struct fscache_object *object)
 541 {
 542         struct fscache_object *p;
 543         struct fscache_cache *cache = object->cache;
 544         int ret;
 545
 546         _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
 547
 548         ASSERTCMP(object->cookie, ==, cookie);
 549
 550         spin_lock(&cookie->lock);
 551
 552         /* there may be multiple initial creations of this object, but we only
 553          * want one */
 554         ret = -EEXIST;
 555         hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {
 556                 if (p->cache == object->cache) {
 557                         if (fscache_object_is_dying(p))
 558                                 ret = -ENOBUFS;
 559                         goto cant_attach_object;
 560                 }
 561         }
 562
 563         /* pin the parent object */
 564         spin_lock_nested(&cookie->parent->lock, 1);
 565         hlist_for_each_entry(p, &cookie->parent->backing_objects,
 566                              cookie_link) {
 567                 if (p->cache == object->cache) {
 568                         if (fscache_object_is_dying(p)) {
 569                                 ret = -ENOBUFS;
 570                                 spin_unlock(&cookie->parent->lock);
 571                                 goto cant_attach_object;
 572                         }
 573                         object->parent = p;
 574                         spin_lock(&p->lock);
 575                         p->n_children++;
 576                         spin_unlock(&p->lock);
 577                         break;
 578                 }
 579         }
 580         spin_unlock(&cookie->parent->lock);
 581
 582         /* attach to the cache's object list */
 583         if (list_empty(&object->cache_link)) {
 584                 spin_lock(&cache->object_list_lock);
 585                 list_add(&object->cache_link, &cache->object_list);
 586                 spin_unlock(&cache->object_list_lock);
 587         }
 588
 589         /* Attach to the cookie.  The object already has a ref on it. */
 590         hlist_add_head(&object->cookie_link, &cookie->backing_objects);
 591
 592         fscache_objlist_add(object);
 593         ret = 0;
 594
 595 cant_attach_object:
 596         spin_unlock(&cookie->lock);
 597         _leave(" = %d", ret);
 598         return ret;
 599 }
 600
 601 /*
 602  * Invalidate an object.  Callable with spinlocks held.
 603  */
 604 void __fscache_invalidate(struct fscache_cookie *cookie)
 605 {
 606         struct fscache_object *object;
 607
 608         _enter("{%s}", cookie->def->name);
 609
 610         fscache_stat(&fscache_n_invalidates);
 611
 612         /* Only permit invalidation of data files.  Invalidating an index will
 613          * require the caller to release all its attachments to the tree rooted
 614          * there, and if it's doing that, it may as well just retire the
 615          * cookie.
 616          */
 617         ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 618
 619         /* If there's an object, we tell the object state machine to handle the
 620          * invalidation on our behalf, otherwise there's nothing to do.
 621          */
 622         if (!hlist_empty(&cookie->backing_objects)) {
 623                 spin_lock(&cookie->lock);
 624
 625                 if (fscache_cookie_enabled(cookie) &&
 626                     !hlist_empty(&cookie->backing_objects) &&
 627                     !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
 628                                       &cookie->flags)) {
 629                         object = hlist_entry(cookie->backing_objects.first,
 630                                              struct fscache_object,
 631                                              cookie_link);
 632                         if (fscache_object_is_live(object))
 633                                 fscache_raise_event(
 634                                         object, FSCACHE_OBJECT_EV_INVALIDATE);
 635                 }
 636
 637                 spin_unlock(&cookie->lock);
 638         }
 639
 640         _leave("");
 641 }
 642 EXPORT_SYMBOL(__fscache_invalidate);
 643
 644 /*
 645  * Wait for object invalidation to complete.
 646  */
 647 void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
 648 {
 649         _enter("%p", cookie);
 650
 651         wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
 652                     TASK_UNINTERRUPTIBLE);
 653
 654         _leave("");
 655 }
 656 EXPORT_SYMBOL(__fscache_wait_on_invalidate);
 657
 658 /*
 659  * update the index entries backing a cookie
 660  */
 661 void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)
 662 {
 663         struct fscache_object *object;
 664
 665         fscache_stat(&fscache_n_updates);
 666
 667         if (!cookie) {
 668                 fscache_stat(&fscache_n_updates_null);
 669                 _leave(" [no cookie]");
 670                 return;
 671         }
 672
 673         _enter("{%s}", cookie->def->name);
 674
 675         spin_lock(&cookie->lock);
 676
 677         fscache_update_aux(cookie, aux_data);
 678
 679         if (fscache_cookie_enabled(cookie)) {
 680                 /* update the index entry on disk in each cache backing this
 681                  * cookie.
 682                  */
 683                 hlist_for_each_entry(object,
 684                                      &cookie->backing_objects, cookie_link) {
 685                         fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
 686                 }
 687         }
 688
 689         spin_unlock(&cookie->lock);
 690         _leave("");
 691 }
 692 EXPORT_SYMBOL(__fscache_update_cookie);
 693
 694 /*
 695  * Disable a cookie to stop it from accepting new requests from the netfs.
 696  */
 697 void __fscache_disable_cookie(struct fscache_cookie *cookie,
 698                               const void *aux_data,
 699                               bool invalidate)
 700 {
 701         struct fscache_object *object;
 702         bool awaken = false;
 703
 704         _enter("%p,%u", cookie, invalidate);
 705
 706         trace_fscache_disable(cookie);
 707
 708         ASSERTCMP(atomic_read(&cookie->n_active), >, 0);
 709
 710         if (atomic_read(&cookie->n_children) != 0) {
 711                 pr_err("Cookie '%s' still has children\n",
 712                        cookie->def->name);
 713                 BUG();
 714         }
 715
 716         wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 717                          TASK_UNINTERRUPTIBLE);
 718
 719         fscache_update_aux(cookie, aux_data);
 720
 721         if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 722                 goto out_unlock_enable;
 723
 724         /* If the cookie is being invalidated, wait for that to complete first
 725          * so that we can reuse the flag.
 726          */
 727         __fscache_wait_on_invalidate(cookie);
 728
 729         /* Dispose of the backing objects */
 730         set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);
 731
 732         spin_lock(&cookie->lock);
 733         if (!hlist_empty(&cookie->backing_objects)) {
 734                 hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {
 735                         if (invalidate)
 736                                 set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
 737                         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 738                         fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL);
 739                 }
 740         } else {
 741                 if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
 742                         awaken = true;
 743         }
 744         spin_unlock(&cookie->lock);
 745         if (awaken)
 746                 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
 747
 748         /* Wait for cessation of activity requiring access to the netfs (when
 749          * n_active reaches 0).  This makes sure outstanding reads and writes
 750          * have completed.
 751          */
 752         if (!atomic_dec_and_test(&cookie->n_active)) {
 753                 wait_var_event(&cookie->n_active,
 754                                !atomic_read(&cookie->n_active));
 755         }
 756
 757         /* Make sure any pending writes are cancelled. */
 758         if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)
 759                 fscache_invalidate_writes(cookie);
 760
 761         /* Reset the cookie state if it wasn't relinquished */
 762         if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {
 763                 atomic_inc(&cookie->n_active);
 764                 set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 765         }
 766
 767 out_unlock_enable:
 768         clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 769         wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 770         _leave("");
 771 }
 772 EXPORT_SYMBOL(__fscache_disable_cookie);
 773
 774 /*
 775  * release a cookie back to the cache
 776  * - the object will be marked as recyclable on disk if retire is true
 777  * - all dependents of this cookie must have already been unregistered
 778  *   (indices/files/pages)
 779  */
 780 void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
 781                                  const void *aux_data,
 782                                  bool retire)
 783 {
 784         fscache_stat(&fscache_n_relinquishes);
 785         if (retire)
 786                 fscache_stat(&fscache_n_relinquishes_retire);
 787
 788         if (!cookie) {
 789                 fscache_stat(&fscache_n_relinquishes_null);
 790                 _leave(" [no cookie]");
 791                 return;
 792         }
 793
 794         _enter("%p{%s,%p,%d},%d",
 795                cookie, cookie->def->name, cookie->netfs_data,
 796                atomic_read(&cookie->n_active), retire);
 797
 798         trace_fscache_relinquish(cookie, retire);
 799
 800         /* No further netfs-accessing operations on this cookie permitted */
 801         if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))
 802                 BUG();
 803
 804         __fscache_disable_cookie(cookie, aux_data, retire);
 805
 806         /* Clear pointers back to the netfs */
 807         cookie->netfs_data      = NULL;
 808         cookie->def             = NULL;
 809         BUG_ON(!radix_tree_empty(&cookie->stores));
 810
 811         if (cookie->parent) {
 812                 ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
 813                 ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
 814                 atomic_dec(&cookie->parent->n_children);
 815         }
 816
 817         /* Dispose of the netfs's link to the cookie */
 818         ASSERTCMP(atomic_read(&cookie->usage), >, 0);
 819         fscache_cookie_put(cookie, fscache_cookie_put_relinquish);
 820
 821         _leave("");
 822 }
 823 EXPORT_SYMBOL(__fscache_relinquish_cookie);
 824
 825 /*
 826  * Remove a cookie from the hash table.
 827  */
 828 static void fscache_unhash_cookie(struct fscache_cookie *cookie)
 829 {
 830         struct hlist_bl_head *h;
 831         unsigned int bucket;
 832
 833         bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 834         h = &fscache_cookie_hash[bucket];
 835
 836         hlist_bl_lock(h);
 837         hlist_bl_del(&cookie->hash_link);
 838         hlist_bl_unlock(h);
 839 }
 840
 841 /*
 842  * Drop a reference to a cookie.
 843  */
 844 void fscache_cookie_put(struct fscache_cookie *cookie,
 845                         enum fscache_cookie_trace where)
 846 {
 847         struct fscache_cookie *parent;
 848         int usage;
 849
 850         _enter("%p", cookie);
 851
 852         do {
 853                 usage = atomic_dec_return(&cookie->usage);
 854                 trace_fscache_cookie(cookie, where, usage);
 855
 856                 if (usage > 0)
 857                         return;
 858                 BUG_ON(usage < 0);
 859
 860                 parent = cookie->parent;
 861                 fscache_unhash_cookie(cookie);
 862                 fscache_free_cookie(cookie);
 863
 864                 cookie = parent;
 865                 where = fscache_cookie_put_parent;
 866         } while (cookie);
 867
 868         _leave("");
 869 }
 870
 871 /*
 872  * check the consistency between the netfs inode and the backing cache
 873  *
 874  * NOTE: it only serves no-index type
 875  */
 876 int __fscache_check_consistency(struct fscache_cookie *cookie,
 877                                 const void *aux_data)
 878 {
 879         struct fscache_operation *op;
 880         struct fscache_object *object;
 881         bool wake_cookie = false;
 882         int ret;
 883
 884         _enter("%p,", cookie);
 885
 886         ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 887
 888         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 889                 return -ERESTARTSYS;
 890
 891         if (hlist_empty(&cookie->backing_objects))
 892                 return 0;
 893
 894         op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
 895         if (!op)
 896                 return -ENOMEM;
 897
 898         fscache_operation_init(cookie, op, NULL, NULL, NULL);
 899         op->flags = FSCACHE_OP_MYTHREAD |
 900                 (1 << FSCACHE_OP_WAITING) |
 901                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 902         trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);
 903
 904         spin_lock(&cookie->lock);
 905
 906         fscache_update_aux(cookie, aux_data);
 907
 908         if (!fscache_cookie_enabled(cookie) ||
 909             hlist_empty(&cookie->backing_objects))
 910                 goto inconsistent;
 911         object = hlist_entry(cookie->backing_objects.first,
 912                              struct fscache_object, cookie_link);
 913         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 914                 goto inconsistent;
 915
 916         op->debug_id = atomic_inc_return(&fscache_op_debug_id);
 917
 918         __fscache_use_cookie(cookie);
 919         if (fscache_submit_op(object, op) < 0)
 920                 goto submit_failed;
 921
 922         /* the work queue now carries its own ref on the object */
 923         spin_unlock(&cookie->lock);
 924
 925         ret = fscache_wait_for_operation_activation(object, op, NULL, NULL);
 926         if (ret == 0) {
 927                 /* ask the cache to honour the operation */
 928                 ret = object->cache->ops->check_consistency(op);
 929                 fscache_op_complete(op, false);
 930         } else if (ret == -ENOBUFS) {
 931                 ret = 0;
 932         }
 933
 934         fscache_put_operation(op);
 935         _leave(" = %d", ret);
 936         return ret;
 937
 938 submit_failed:
 939         wake_cookie = __fscache_unuse_cookie(cookie);
 940 inconsistent:
 941         spin_unlock(&cookie->lock);
 942         if (wake_cookie)
 943                 __fscache_wake_unused_cookie(cookie);
 944         kfree(op);
 945         _leave(" = -ESTALE");
 946         return -ESTALE;
 947 }
 948 EXPORT_SYMBOL(__fscache_check_consistency);