fs/fscache/object.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* FS-Cache object state machine handler
   3  *
   4  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  *
   7  * See Documentation/filesystems/caching/object.rst for a description of the
   8  * object state machine and the in-kernel representations.
   9  */
  10
  11 #define FSCACHE_DEBUG_LEVEL COOKIE
  12 #include <linux/module.h>
  13 #include <linux/slab.h>
  14 #include <linux/prefetch.h>
  15 #include "internal.h"
  16
  17 static const struct fscache_state *fscache_abort_initialisation(struct fscache_object *, int);
  18 static const struct fscache_state *fscache_kill_dependents(struct fscache_object *, int);
  19 static const struct fscache_state *fscache_drop_object(struct fscache_object *, int);
  20 static const struct fscache_state *fscache_initialise_object(struct fscache_object *, int);
  21 static const struct fscache_state *fscache_invalidate_object(struct fscache_object *, int);
  22 static const struct fscache_state *fscache_jumpstart_dependents(struct fscache_object *, int);
  23 static const struct fscache_state *fscache_kill_object(struct fscache_object *, int);
  24 static const struct fscache_state *fscache_lookup_failure(struct fscache_object *, int);
  25 static const struct fscache_state *fscache_look_up_object(struct fscache_object *, int);
  26 static const struct fscache_state *fscache_object_available(struct fscache_object *, int);
  27 static const struct fscache_state *fscache_parent_ready(struct fscache_object *, int);
  28 static const struct fscache_state *fscache_update_object(struct fscache_object *, int);
  29 static const struct fscache_state *fscache_object_dead(struct fscache_object *, int);
  30
  31 #define __STATE_NAME(n) fscache_osm_##n
  32 #define STATE(n) (&__STATE_NAME(n))
  33
  34 /*
  35  * Define a work state.  Work states are execution states.  No event processing
  36  * is performed by them.  The function attached to a work state returns a
  37  * pointer indicating the next state to which the state machine should
  38  * transition.  Returning NO_TRANSIT repeats the current state, but goes back
  39  * to the scheduler first.
  40  */
  41 #define WORK_STATE(n, sn, f) \
  42         const struct fscache_state __STATE_NAME(n) = {                  \
  43                 .name = #n,                                             \
  44                 .short_name = sn,                                       \
  45                 .work = f                                               \
  46         }
  47
  48 /*
  49  * Returns from work states.
  50  */
  51 #define transit_to(state) ({ prefetch(&STATE(state)->work); STATE(state); })
  52
  53 #define NO_TRANSIT ((struct fscache_state *)NULL)
  54
  55 /*
  56  * Define a wait state.  Wait states are event processing states.  No execution
  57  * is performed by them.  Wait states are just tables of "if event X occurs,
  58  * clear it and transition to state Y".  The dispatcher returns to the
  59  * scheduler if none of the events in which the wait state has an interest are
  60  * currently pending.
  61  */
  62 #define WAIT_STATE(n, sn, ...) \
  63         const struct fscache_state __STATE_NAME(n) = {                  \
  64                 .name = #n,                                             \
  65                 .short_name = sn,                                       \
  66                 .work = NULL,                                           \
  67                 .transitions = { __VA_ARGS__, { 0, NULL } }             \
  68         }
  69
  70 #define TRANSIT_TO(state, emask) \
  71         { .events = (emask), .transit_to = STATE(state) }
  72
  73 /*
  74  * The object state machine.
  75  */
  76 static WORK_STATE(INIT_OBJECT,          "INIT", fscache_initialise_object);
  77 static WORK_STATE(PARENT_READY,         "PRDY", fscache_parent_ready);
  78 static WORK_STATE(ABORT_INIT,           "ABRT", fscache_abort_initialisation);
  79 static WORK_STATE(LOOK_UP_OBJECT,       "LOOK", fscache_look_up_object);
  80 static WORK_STATE(OBJECT_AVAILABLE,     "AVBL", fscache_object_available);
  81 static WORK_STATE(JUMPSTART_DEPS,       "JUMP", fscache_jumpstart_dependents);
  82
  83 static WORK_STATE(INVALIDATE_OBJECT,    "INVL", fscache_invalidate_object);
  84 static WORK_STATE(UPDATE_OBJECT,        "UPDT", fscache_update_object);
  85
  86 static WORK_STATE(LOOKUP_FAILURE,       "LCFL", fscache_lookup_failure);
  87 static WORK_STATE(KILL_OBJECT,          "KILL", fscache_kill_object);
  88 static WORK_STATE(KILL_DEPENDENTS,      "KDEP", fscache_kill_dependents);
  89 static WORK_STATE(DROP_OBJECT,          "DROP", fscache_drop_object);
  90 static WORK_STATE(OBJECT_DEAD,          "DEAD", fscache_object_dead);
  91
  92 static WAIT_STATE(WAIT_FOR_INIT,        "?INI",
  93                   TRANSIT_TO(INIT_OBJECT,       1 << FSCACHE_OBJECT_EV_NEW_CHILD));
  94
  95 static WAIT_STATE(WAIT_FOR_PARENT,      "?PRN",
  96                   TRANSIT_TO(PARENT_READY,      1 << FSCACHE_OBJECT_EV_PARENT_READY));
  97
  98 static WAIT_STATE(WAIT_FOR_CMD,         "?CMD",
  99                   TRANSIT_TO(INVALIDATE_OBJECT, 1 << FSCACHE_OBJECT_EV_INVALIDATE),
 100                   TRANSIT_TO(UPDATE_OBJECT,     1 << FSCACHE_OBJECT_EV_UPDATE),
 101                   TRANSIT_TO(JUMPSTART_DEPS,    1 << FSCACHE_OBJECT_EV_NEW_CHILD));
 102
 103 static WAIT_STATE(WAIT_FOR_CLEARANCE,   "?CLR",
 104                   TRANSIT_TO(KILL_OBJECT,       1 << FSCACHE_OBJECT_EV_CLEARED));
 105
 106 /*
 107  * Out-of-band event transition tables.  These are for handling unexpected
 108  * events, such as an I/O error.  If an OOB event occurs, the state machine
 109  * clears and disables the event and forces a transition to the nominated work
 110  * state (acurrently executing work states will complete first).
 111  *
 112  * In such a situation, object->state remembers the state the machine should
 113  * have been in/gone to and returning NO_TRANSIT returns to that.
 114  */
 115 static const struct fscache_transition fscache_osm_init_oob[] = {
 116            TRANSIT_TO(ABORT_INIT,
 117                       (1 << FSCACHE_OBJECT_EV_ERROR) |
 118                       (1 << FSCACHE_OBJECT_EV_KILL)),
 119            { 0, NULL }
 120 };
 121
 122 static const struct fscache_transition fscache_osm_lookup_oob[] = {
 123            TRANSIT_TO(LOOKUP_FAILURE,
 124                       (1 << FSCACHE_OBJECT_EV_ERROR) |
 125                       (1 << FSCACHE_OBJECT_EV_KILL)),
 126            { 0, NULL }
 127 };
 128
 129 static const struct fscache_transition fscache_osm_run_oob[] = {
 130            TRANSIT_TO(KILL_OBJECT,
 131                       (1 << FSCACHE_OBJECT_EV_ERROR) |
 132                       (1 << FSCACHE_OBJECT_EV_KILL)),
 133            { 0, NULL }
 134 };
 135
 136 static int  fscache_get_object(struct fscache_object *,
 137                                enum fscache_obj_ref_trace);
 138 static void fscache_put_object(struct fscache_object *,
 139                                enum fscache_obj_ref_trace);
 140 static bool fscache_enqueue_dependents(struct fscache_object *, int);
 141 static void fscache_dequeue_object(struct fscache_object *);
 142 static void fscache_update_aux_data(struct fscache_object *);
 143
 144 /*
 145  * we need to notify the parent when an op completes that we had outstanding
 146  * upon it
 147  */
 148 static inline void fscache_done_parent_op(struct fscache_object *object)
 149 {
 150         struct fscache_object *parent = object->parent;
 151
 152         _enter("OBJ%x {OBJ%x,%x}",
 153                object->debug_id, parent->debug_id, parent->n_ops);
 154
 155         spin_lock_nested(&parent->lock, 1);
 156         parent->n_obj_ops--;
 157         parent->n_ops--;
 158         if (parent->n_ops == 0)
 159                 fscache_raise_event(parent, FSCACHE_OBJECT_EV_CLEARED);
 160         spin_unlock(&parent->lock);
 161 }
 162
 163 /*
 164  * Object state machine dispatcher.
 165  */
 166 static void fscache_object_sm_dispatcher(struct fscache_object *object)
 167 {
 168         const struct fscache_transition *t;
 169         const struct fscache_state *state, *new_state;
 170         unsigned long events, event_mask;
 171         bool oob;
 172         int event = -1;
 173
 174         ASSERT(object != NULL);
 175
 176         _enter("{OBJ%x,%s,%lx}",
 177                object->debug_id, object->state->name, object->events);
 178
 179         event_mask = object->event_mask;
 180 restart:
 181         object->event_mask = 0; /* Mask normal event handling */
 182         state = object->state;
 183 restart_masked:
 184         events = object->events;
 185
 186         /* Handle any out-of-band events (typically an error) */
 187         if (events & object->oob_event_mask) {
 188                 _debug("{OBJ%x} oob %lx",
 189                        object->debug_id, events & object->oob_event_mask);
 190                 oob = true;
 191                 for (t = object->oob_table; t->events; t++) {
 192                         if (events & t->events) {
 193                                 state = t->transit_to;
 194                                 ASSERT(state->work != NULL);
 195                                 event = fls(events & t->events) - 1;
 196                                 __clear_bit(event, &object->oob_event_mask);
 197                                 clear_bit(event, &object->events);
 198                                 goto execute_work_state;
 199                         }
 200                 }
 201         }
 202         oob = false;
 203
 204         /* Wait states are just transition tables */
 205         if (!state->work) {
 206                 if (events & event_mask) {
 207                         for (t = state->transitions; t->events; t++) {
 208                                 if (events & t->events) {
 209                                         new_state = t->transit_to;
 210                                         event = fls(events & t->events) - 1;
 211                                         trace_fscache_osm(object, state,
 212                                                           true, false, event);
 213                                         clear_bit(event, &object->events);
 214                                         _debug("{OBJ%x} ev %d: %s -> %s",
 215                                                object->debug_id, event,
 216                                                state->name, new_state->name);
 217                                         object->state = state = new_state;
 218                                         goto execute_work_state;
 219                                 }
 220                         }
 221
 222                         /* The event mask didn't include all the tabled bits */
 223                         BUG();
 224                 }
 225                 /* Randomly woke up */
 226                 goto unmask_events;
 227         }
 228
 229 execute_work_state:
 230         _debug("{OBJ%x} exec %s", object->debug_id, state->name);
 231
 232         trace_fscache_osm(object, state, false, oob, event);
 233         new_state = state->work(object, event);
 234         event = -1;
 235         if (new_state == NO_TRANSIT) {
 236                 _debug("{OBJ%x} %s notrans", object->debug_id, state->name);
 237                 if (unlikely(state == STATE(OBJECT_DEAD))) {
 238                         _leave(" [dead]");
 239                         return;
 240                 }
 241                 fscache_enqueue_object(object);
 242                 event_mask = object->oob_event_mask;
 243                 goto unmask_events;
 244         }
 245
 246         _debug("{OBJ%x} %s -> %s",
 247                object->debug_id, state->name, new_state->name);
 248         object->state = state = new_state;
 249
 250         if (state->work) {
 251                 if (unlikely(state == STATE(OBJECT_DEAD))) {
 252                         _leave(" [dead]");
 253                         return;
 254                 }
 255                 goto restart_masked;
 256         }
 257
 258         /* Transited to wait state */
 259         event_mask = object->oob_event_mask;
 260         for (t = state->transitions; t->events; t++)
 261                 event_mask |= t->events;
 262
 263 unmask_events:
 264         object->event_mask = event_mask;
 265         smp_mb();
 266         events = object->events;
 267         if (events & event_mask)
 268                 goto restart;
 269         _leave(" [msk %lx]", event_mask);
 270 }
 271
 272 /*
 273  * execute an object
 274  */
 275 static void fscache_object_work_func(struct work_struct *work)
 276 {
 277         struct fscache_object *object =
 278                 container_of(work, struct fscache_object, work);
 279
 280         _enter("{OBJ%x}", object->debug_id);
 281
 282         fscache_object_sm_dispatcher(object);
 283         fscache_put_object(object, fscache_obj_put_work);
 284 }
 285
 286 /**
 287  * fscache_object_init - Initialise a cache object description
 288  * @object: Object description
 289  * @cookie: Cookie object will be attached to
 290  * @cache: Cache in which backing object will be found
 291  *
 292  * Initialise a cache object description to its basic values.
 293  *
 294  * See Documentation/filesystems/caching/backend-api.rst for a complete
 295  * description.
 296  */
 297 void fscache_object_init(struct fscache_object *object,
 298                          struct fscache_cookie *cookie,
 299                          struct fscache_cache *cache)
 300 {
 301         const struct fscache_transition *t;
 302
 303         atomic_inc(&cache->object_count);
 304
 305         object->state = STATE(WAIT_FOR_INIT);
 306         object->oob_table = fscache_osm_init_oob;
 307         object->flags = 1 << FSCACHE_OBJECT_IS_LIVE;
 308         spin_lock_init(&object->lock);
 309         INIT_LIST_HEAD(&object->cache_link);
 310         INIT_HLIST_NODE(&object->cookie_link);
 311         INIT_WORK(&object->work, fscache_object_work_func);
 312         INIT_LIST_HEAD(&object->dependents);
 313         INIT_LIST_HEAD(&object->dep_link);
 314         INIT_LIST_HEAD(&object->pending_ops);
 315         object->n_children = 0;
 316         object->n_ops = object->n_in_progress = object->n_exclusive = 0;
 317         object->events = 0;
 318         object->store_limit = 0;
 319         object->store_limit_l = 0;
 320         object->cache = cache;
 321         object->cookie = cookie;
 322         fscache_cookie_get(cookie, fscache_cookie_get_attach_object);
 323         object->parent = NULL;
 324 #ifdef CONFIG_FSCACHE_OBJECT_LIST
 325         RB_CLEAR_NODE(&object->objlist_link);
 326 #endif
 327
 328         object->oob_event_mask = 0;
 329         for (t = object->oob_table; t->events; t++)
 330                 object->oob_event_mask |= t->events;
 331         object->event_mask = object->oob_event_mask;
 332         for (t = object->state->transitions; t->events; t++)
 333                 object->event_mask |= t->events;
 334 }
 335 EXPORT_SYMBOL(fscache_object_init);
 336
 337 /*
 338  * Mark the object as no longer being live, making sure that we synchronise
 339  * against op submission.
 340  */
 341 static inline void fscache_mark_object_dead(struct fscache_object *object)
 342 {
 343         spin_lock(&object->lock);
 344         clear_bit(FSCACHE_OBJECT_IS_LIVE, &object->flags);
 345         spin_unlock(&object->lock);
 346 }
 347
 348 /*
 349  * Abort object initialisation before we start it.
 350  */
 351 static const struct fscache_state *fscache_abort_initialisation(struct fscache_object *object,
 352                                                                 int event)
 353 {
 354         _enter("{OBJ%x},%d", object->debug_id, event);
 355
 356         object->oob_event_mask = 0;
 357         fscache_dequeue_object(object);
 358         return transit_to(KILL_OBJECT);
 359 }
 360
 361 /*
 362  * initialise an object
 363  * - check the specified object's parent to see if we can make use of it
 364  *   immediately to do a creation
 365  * - we may need to start the process of creating a parent and we need to wait
 366  *   for the parent's lookup and creation to complete if it's not there yet
 367  */
 368 static const struct fscache_state *fscache_initialise_object(struct fscache_object *object,
 369                                                              int event)
 370 {
 371         struct fscache_object *parent;
 372         bool success;
 373
 374         _enter("{OBJ%x},%d", object->debug_id, event);
 375
 376         ASSERT(list_empty(&object->dep_link));
 377
 378         parent = object->parent;
 379         if (!parent) {
 380                 _leave(" [no parent]");
 381                 return transit_to(DROP_OBJECT);
 382         }
 383
 384         _debug("parent: %s of:%lx", parent->state->name, parent->flags);
 385
 386         if (fscache_object_is_dying(parent)) {
 387                 _leave(" [bad parent]");
 388                 return transit_to(DROP_OBJECT);
 389         }
 390
 391         if (fscache_object_is_available(parent)) {
 392                 _leave(" [ready]");
 393                 return transit_to(PARENT_READY);
 394         }
 395
 396         _debug("wait");
 397
 398         spin_lock(&parent->lock);
 399         fscache_stat(&fscache_n_cop_grab_object);
 400         success = false;
 401         if (fscache_object_is_live(parent) &&
 402             object->cache->ops->grab_object(object, fscache_obj_get_add_to_deps)) {
 403                 list_add(&object->dep_link, &parent->dependents);
 404                 success = true;
 405         }
 406         fscache_stat_d(&fscache_n_cop_grab_object);
 407         spin_unlock(&parent->lock);
 408         if (!success) {
 409                 _leave(" [grab failed]");
 410                 return transit_to(DROP_OBJECT);
 411         }
 412
 413         /* fscache_acquire_non_index_cookie() uses this
 414          * to wake the chain up */
 415         fscache_raise_event(parent, FSCACHE_OBJECT_EV_NEW_CHILD);
 416         _leave(" [wait]");
 417         return transit_to(WAIT_FOR_PARENT);
 418 }
 419
 420 /*
 421  * Once the parent object is ready, we should kick off our lookup op.
 422  */
 423 static const struct fscache_state *fscache_parent_ready(struct fscache_object *object,
 424                                                         int event)
 425 {
 426         struct fscache_object *parent = object->parent;
 427
 428         _enter("{OBJ%x},%d", object->debug_id, event);
 429
 430         ASSERT(parent != NULL);
 431
 432         spin_lock(&parent->lock);
 433         parent->n_ops++;
 434         parent->n_obj_ops++;
 435         spin_unlock(&parent->lock);
 436
 437         _leave("");
 438         return transit_to(LOOK_UP_OBJECT);
 439 }
 440
 441 /*
 442  * look an object up in the cache from which it was allocated
 443  * - we hold an "access lock" on the parent object, so the parent object cannot
 444  *   be withdrawn by either party till we've finished
 445  */
 446 static const struct fscache_state *fscache_look_up_object(struct fscache_object *object,
 447                                                           int event)
 448 {
 449         struct fscache_cookie *cookie = object->cookie;
 450         struct fscache_object *parent = object->parent;
 451         int ret;
 452
 453         _enter("{OBJ%x},%d", object->debug_id, event);
 454
 455         object->oob_table = fscache_osm_lookup_oob;
 456
 457         ASSERT(parent != NULL);
 458         ASSERTCMP(parent->n_ops, >, 0);
 459         ASSERTCMP(parent->n_obj_ops, >, 0);
 460
 461         /* make sure the parent is still available */
 462         ASSERT(fscache_object_is_available(parent));
 463
 464         if (fscache_object_is_dying(parent) ||
 465             test_bit(FSCACHE_IOERROR, &object->cache->flags) ||
 466             !fscache_use_cookie(object)) {
 467                 _leave(" [unavailable]");
 468                 return transit_to(LOOKUP_FAILURE);
 469         }
 470
 471         _debug("LOOKUP \"%s\" in \"%s\"",
 472                cookie->def->name, object->cache->tag->name);
 473
 474         fscache_stat(&fscache_n_object_lookups);
 475         fscache_stat(&fscache_n_cop_lookup_object);
 476         ret = object->cache->ops->lookup_object(object);
 477         fscache_stat_d(&fscache_n_cop_lookup_object);
 478
 479         fscache_unuse_cookie(object);
 480
 481         if (ret == -ETIMEDOUT) {
 482                 /* probably stuck behind another object, so move this one to
 483                  * the back of the queue */
 484                 fscache_stat(&fscache_n_object_lookups_timed_out);
 485                 _leave(" [timeout]");
 486                 return NO_TRANSIT;
 487         }
 488
 489         if (ret < 0) {
 490                 _leave(" [error]");
 491                 return transit_to(LOOKUP_FAILURE);
 492         }
 493
 494         _leave(" [ok]");
 495         return transit_to(OBJECT_AVAILABLE);
 496 }
 497
 498 /**
 499  * fscache_object_lookup_negative - Note negative cookie lookup
 500  * @object: Object pointing to cookie to mark
 501  *
 502  * Note negative lookup, permitting those waiting to read data from an already
 503  * existing backing object to continue as there's no data for them to read.
 504  */
 505 void fscache_object_lookup_negative(struct fscache_object *object)
 506 {
 507         struct fscache_cookie *cookie = object->cookie;
 508
 509         _enter("{OBJ%x,%s}", object->debug_id, object->state->name);
 510
 511         if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) {
 512                 fscache_stat(&fscache_n_object_lookups_negative);
 513
 514                 /* Allow write requests to begin stacking up and read requests to begin
 515                  * returning ENODATA.
 516                  */
 517                 set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 518                 clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
 519
 520                 clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
 521                 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);
 522         }
 523         _leave("");
 524 }
 525 EXPORT_SYMBOL(fscache_object_lookup_negative);
 526
 527 /**
 528  * fscache_obtained_object - Note successful object lookup or creation
 529  * @object: Object pointing to cookie to mark
 530  *
 531  * Note successful lookup and/or creation, permitting those waiting to write
 532  * data to a backing object to continue.
 533  *
 534  * Note that after calling this, an object's cookie may be relinquished by the
 535  * netfs, and so must be accessed with object lock held.
 536  */
 537 void fscache_obtained_object(struct fscache_object *object)
 538 {
 539         struct fscache_cookie *cookie = object->cookie;
 540
 541         _enter("{OBJ%x,%s}", object->debug_id, object->state->name);
 542
 543         /* if we were still looking up, then we must have a positive lookup
 544          * result, in which case there may be data available */
 545         if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) {
 546                 fscache_stat(&fscache_n_object_lookups_positive);
 547
 548                 /* We do (presumably) have data */
 549                 clear_bit_unlock(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 550                 clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
 551
 552                 /* Allow write requests to begin stacking up and read requests
 553                  * to begin shovelling data.
 554                  */
 555                 clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
 556                 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);
 557         } else {
 558                 fscache_stat(&fscache_n_object_created);
 559         }
 560
 561         set_bit(FSCACHE_OBJECT_IS_AVAILABLE, &object->flags);
 562         _leave("");
 563 }
 564 EXPORT_SYMBOL(fscache_obtained_object);
 565
 566 /*
 567  * handle an object that has just become available
 568  */
 569 static const struct fscache_state *fscache_object_available(struct fscache_object *object,
 570                                                             int event)
 571 {
 572         _enter("{OBJ%x},%d", object->debug_id, event);
 573
 574         object->oob_table = fscache_osm_run_oob;
 575
 576         spin_lock(&object->lock);
 577
 578         fscache_done_parent_op(object);
 579         if (object->n_in_progress == 0) {
 580                 if (object->n_ops > 0) {
 581                         ASSERTCMP(object->n_ops, >=, object->n_obj_ops);
 582                         fscache_start_operations(object);
 583                 } else {
 584                         ASSERT(list_empty(&object->pending_ops));
 585                 }
 586         }
 587         spin_unlock(&object->lock);
 588
 589         fscache_stat(&fscache_n_cop_lookup_complete);
 590         object->cache->ops->lookup_complete(object);
 591         fscache_stat_d(&fscache_n_cop_lookup_complete);
 592
 593         fscache_stat(&fscache_n_object_avail);
 594
 595         _leave("");
 596         return transit_to(JUMPSTART_DEPS);
 597 }
 598
 599 /*
 600  * Wake up this object's dependent objects now that we've become available.
 601  */
 602 static const struct fscache_state *fscache_jumpstart_dependents(struct fscache_object *object,
 603                                                                 int event)
 604 {
 605         _enter("{OBJ%x},%d", object->debug_id, event);
 606
 607         if (!fscache_enqueue_dependents(object, FSCACHE_OBJECT_EV_PARENT_READY))
 608                 return NO_TRANSIT; /* Not finished; requeue */
 609         return transit_to(WAIT_FOR_CMD);
 610 }
 611
 612 /*
 613  * Handle lookup or creation failute.
 614  */
 615 static const struct fscache_state *fscache_lookup_failure(struct fscache_object *object,
 616                                                           int event)
 617 {
 618         struct fscache_cookie *cookie;
 619
 620         _enter("{OBJ%x},%d", object->debug_id, event);
 621
 622         object->oob_event_mask = 0;
 623
 624         fscache_stat(&fscache_n_cop_lookup_complete);
 625         object->cache->ops->lookup_complete(object);
 626         fscache_stat_d(&fscache_n_cop_lookup_complete);
 627
 628         set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags);
 629
 630         cookie = object->cookie;
 631         set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
 632         if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags))
 633                 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);
 634
 635         fscache_done_parent_op(object);
 636         return transit_to(KILL_OBJECT);
 637 }
 638
 639 /*
 640  * Wait for completion of all active operations on this object and the death of
 641  * all child objects of this object.
 642  */
 643 static const struct fscache_state *fscache_kill_object(struct fscache_object *object,
 644                                                        int event)
 645 {
 646         _enter("{OBJ%x,%d,%d},%d",
 647                object->debug_id, object->n_ops, object->n_children, event);
 648
 649         fscache_mark_object_dead(object);
 650         object->oob_event_mask = 0;
 651
 652         if (test_bit(FSCACHE_OBJECT_RETIRED, &object->flags)) {
 653                 /* Reject any new read/write ops and abort any that are pending. */
 654                 clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 655                 fscache_cancel_all_ops(object);
 656         }
 657
 658         if (list_empty(&object->dependents) &&
 659             object->n_ops == 0 &&
 660             object->n_children == 0)
 661                 return transit_to(DROP_OBJECT);
 662
 663         if (object->n_in_progress == 0) {
 664                 spin_lock(&object->lock);
 665                 if (object->n_ops > 0 && object->n_in_progress == 0)
 666                         fscache_start_operations(object);
 667                 spin_unlock(&object->lock);
 668         }
 669
 670         if (!list_empty(&object->dependents))
 671                 return transit_to(KILL_DEPENDENTS);
 672
 673         return transit_to(WAIT_FOR_CLEARANCE);
 674 }
 675
 676 /*
 677  * Kill dependent objects.
 678  */
 679 static const struct fscache_state *fscache_kill_dependents(struct fscache_object *object,
 680                                                            int event)
 681 {
 682         _enter("{OBJ%x},%d", object->debug_id, event);
 683
 684         if (!fscache_enqueue_dependents(object, FSCACHE_OBJECT_EV_KILL))
 685                 return NO_TRANSIT; /* Not finished */
 686         return transit_to(WAIT_FOR_CLEARANCE);
 687 }
 688
 689 /*
 690  * Drop an object's attachments
 691  */
 692 static const struct fscache_state *fscache_drop_object(struct fscache_object *object,
 693                                                        int event)
 694 {
 695         struct fscache_object *parent = object->parent;
 696         struct fscache_cookie *cookie = object->cookie;
 697         struct fscache_cache *cache = object->cache;
 698         bool awaken = false;
 699
 700         _enter("{OBJ%x,%d},%d", object->debug_id, object->n_children, event);
 701
 702         ASSERT(cookie != NULL);
 703         ASSERT(!hlist_unhashed(&object->cookie_link));
 704
 705         if (test_bit(FSCACHE_COOKIE_AUX_UPDATED, &cookie->flags)) {
 706                 _debug("final update");
 707                 fscache_update_aux_data(object);
 708         }
 709
 710         /* Make sure the cookie no longer points here and that the netfs isn't
 711          * waiting for us.
 712          */
 713         spin_lock(&cookie->lock);
 714         hlist_del_init(&object->cookie_link);
 715         if (hlist_empty(&cookie->backing_objects) &&
 716             test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
 717                 awaken = true;
 718         spin_unlock(&cookie->lock);
 719
 720         if (awaken)
 721                 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
 722         if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags))
 723                 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);
 724
 725
 726         /* Prevent a race with our last child, which has to signal EV_CLEARED
 727          * before dropping our spinlock.
 728          */
 729         spin_lock(&object->lock);
 730         spin_unlock(&object->lock);
 731
 732         /* Discard from the cache's collection of objects */
 733         spin_lock(&cache->object_list_lock);
 734         list_del_init(&object->cache_link);
 735         spin_unlock(&cache->object_list_lock);
 736
 737         fscache_stat(&fscache_n_cop_drop_object);
 738         cache->ops->drop_object(object);
 739         fscache_stat_d(&fscache_n_cop_drop_object);
 740
 741         /* The parent object wants to know when all it dependents have gone */
 742         if (parent) {
 743                 _debug("release parent OBJ%x {%d}",
 744                        parent->debug_id, parent->n_children);
 745
 746                 spin_lock(&parent->lock);
 747                 parent->n_children--;
 748                 if (parent->n_children == 0)
 749                         fscache_raise_event(parent, FSCACHE_OBJECT_EV_CLEARED);
 750                 spin_unlock(&parent->lock);
 751                 object->parent = NULL;
 752         }
 753
 754         /* this just shifts the object release to the work processor */
 755         fscache_put_object(object, fscache_obj_put_drop_obj);
 756         fscache_stat(&fscache_n_object_dead);
 757
 758         _leave("");
 759         return transit_to(OBJECT_DEAD);
 760 }
 761
 762 /*
 763  * get a ref on an object
 764  */
 765 static int fscache_get_object(struct fscache_object *object,
 766                               enum fscache_obj_ref_trace why)
 767 {
 768         int ret;
 769
 770         fscache_stat(&fscache_n_cop_grab_object);
 771         ret = object->cache->ops->grab_object(object, why) ? 0 : -EAGAIN;
 772         fscache_stat_d(&fscache_n_cop_grab_object);
 773         return ret;
 774 }
 775
 776 /*
 777  * Discard a ref on an object
 778  */
 779 static void fscache_put_object(struct fscache_object *object,
 780                                enum fscache_obj_ref_trace why)
 781 {
 782         fscache_stat(&fscache_n_cop_put_object);
 783         object->cache->ops->put_object(object, why);
 784         fscache_stat_d(&fscache_n_cop_put_object);
 785 }
 786
 787 /**
 788  * fscache_object_destroy - Note that a cache object is about to be destroyed
 789  * @object: The object to be destroyed
 790  *
 791  * Note the imminent destruction and deallocation of a cache object record.
 792  */
 793 void fscache_object_destroy(struct fscache_object *object)
 794 {
 795         /* We can get rid of the cookie now */
 796         fscache_cookie_put(object->cookie, fscache_cookie_put_object);
 797         object->cookie = NULL;
 798 }
 799 EXPORT_SYMBOL(fscache_object_destroy);
 800
 801 /*
 802  * enqueue an object for metadata-type processing
 803  */
 804 void fscache_enqueue_object(struct fscache_object *object)
 805 {
 806         _enter("{OBJ%x}", object->debug_id);
 807
 808         if (fscache_get_object(object, fscache_obj_get_queue) >= 0) {
 809                 wait_queue_head_t *cong_wq =
 810                         &get_cpu_var(fscache_object_cong_wait);
 811
 812                 if (queue_work(fscache_object_wq, &object->work)) {
 813                         if (fscache_object_congested())
 814                                 wake_up(cong_wq);
 815                 } else
 816                         fscache_put_object(object, fscache_obj_put_queue);
 817
 818                 put_cpu_var(fscache_object_cong_wait);
 819         }
 820 }
 821
 822 /**
 823  * fscache_object_sleep_till_congested - Sleep until object wq is congested
 824  * @timeoutp: Scheduler sleep timeout
 825  *
 826  * Allow an object handler to sleep until the object workqueue is congested.
 827  *
 828  * The caller must set up a wake up event before calling this and must have set
 829  * the appropriate sleep mode (such as TASK_UNINTERRUPTIBLE) and tested its own
 830  * condition before calling this function as no test is made here.
 831  *
 832  * %true is returned if the object wq is congested, %false otherwise.
 833  */
 834 bool fscache_object_sleep_till_congested(signed long *timeoutp)
 835 {
 836         wait_queue_head_t *cong_wq = this_cpu_ptr(&fscache_object_cong_wait);
 837         DEFINE_WAIT(wait);
 838
 839         if (fscache_object_congested())
 840                 return true;
 841
 842         add_wait_queue_exclusive(cong_wq, &wait);
 843         if (!fscache_object_congested())
 844                 *timeoutp = schedule_timeout(*timeoutp);
 845         finish_wait(cong_wq, &wait);
 846
 847         return fscache_object_congested();
 848 }
 849 EXPORT_SYMBOL_GPL(fscache_object_sleep_till_congested);
 850
 851 /*
 852  * Enqueue the dependents of an object for metadata-type processing.
 853  *
 854  * If we don't manage to finish the list before the scheduler wants to run
 855  * again then return false immediately.  We return true if the list was
 856  * cleared.
 857  */
 858 static bool fscache_enqueue_dependents(struct fscache_object *object, int event)
 859 {
 860         struct fscache_object *dep;
 861         bool ret = true;
 862
 863         _enter("{OBJ%x}", object->debug_id);
 864
 865         if (list_empty(&object->dependents))
 866                 return true;
 867
 868         spin_lock(&object->lock);
 869
 870         while (!list_empty(&object->dependents)) {
 871                 dep = list_entry(object->dependents.next,
 872                                  struct fscache_object, dep_link);
 873                 list_del_init(&dep->dep_link);
 874
 875                 fscache_raise_event(dep, event);
 876                 fscache_put_object(dep, fscache_obj_put_enq_dep);
 877
 878                 if (!list_empty(&object->dependents) && need_resched()) {
 879                         ret = false;
 880                         break;
 881                 }
 882         }
 883
 884         spin_unlock(&object->lock);
 885         return ret;
 886 }
 887
 888 /*
 889  * remove an object from whatever queue it's waiting on
 890  */
 891 static void fscache_dequeue_object(struct fscache_object *object)
 892 {
 893         _enter("{OBJ%x}", object->debug_id);
 894
 895         if (!list_empty(&object->dep_link)) {
 896                 spin_lock(&object->parent->lock);
 897                 list_del_init(&object->dep_link);
 898                 spin_unlock(&object->parent->lock);
 899         }
 900
 901         _leave("");
 902 }
 903
 904 /**
 905  * fscache_check_aux - Ask the netfs whether an object on disk is still valid
 906  * @object: The object to ask about
 907  * @data: The auxiliary data for the object
 908  * @datalen: The size of the auxiliary data
 909  * @object_size: The size of the object according to the server.
 910  *
 911  * This function consults the netfs about the coherency state of an object.
 912  * The caller must be holding a ref on cookie->n_active (held by
 913  * fscache_look_up_object() on behalf of the cache backend during object lookup
 914  * and creation).
 915  */
 916 enum fscache_checkaux fscache_check_aux(struct fscache_object *object,
 917                                         const void *data, uint16_t datalen,
 918                                         loff_t object_size)
 919 {
 920         enum fscache_checkaux result;
 921
 922         if (!object->cookie->def->check_aux) {
 923                 fscache_stat(&fscache_n_checkaux_none);
 924                 return FSCACHE_CHECKAUX_OKAY;
 925         }
 926
 927         result = object->cookie->def->check_aux(object->cookie->netfs_data,
 928                                                 data, datalen, object_size);
 929         switch (result) {
 930                 /* entry okay as is */
 931         case FSCACHE_CHECKAUX_OKAY:
 932                 fscache_stat(&fscache_n_checkaux_okay);
 933                 break;
 934
 935                 /* entry requires update */
 936         case FSCACHE_CHECKAUX_NEEDS_UPDATE:
 937                 fscache_stat(&fscache_n_checkaux_update);
 938                 break;
 939
 940                 /* entry requires deletion */
 941         case FSCACHE_CHECKAUX_OBSOLETE:
 942                 fscache_stat(&fscache_n_checkaux_obsolete);
 943                 break;
 944
 945         default:
 946                 BUG();
 947         }
 948
 949         return result;
 950 }
 951 EXPORT_SYMBOL(fscache_check_aux);
 952
 953 /*
 954  * Asynchronously invalidate an object.
 955  */
 956 static const struct fscache_state *_fscache_invalidate_object(struct fscache_object *object,
 957                                                               int event)
 958 {
 959         struct fscache_operation *op;
 960         struct fscache_cookie *cookie = object->cookie;
 961
 962         _enter("{OBJ%x},%d", object->debug_id, event);
 963
 964         /* We're going to need the cookie.  If the cookie is not available then
 965          * retire the object instead.
 966          */
 967         if (!fscache_use_cookie(object)) {
 968                 ASSERT(radix_tree_empty(&object->cookie->stores));
 969                 set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
 970                 _leave(" [no cookie]");
 971                 return transit_to(KILL_OBJECT);
 972         }
 973
 974         /* Reject any new read/write ops and abort any that are pending. */
 975         fscache_invalidate_writes(cookie);
 976         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 977         fscache_cancel_all_ops(object);
 978
 979         /* Now we have to wait for in-progress reads and writes */
 980         op = kzalloc(sizeof(*op), GFP_KERNEL);
 981         if (!op)
 982                 goto nomem;
 983
 984         fscache_operation_init(cookie, op, object->cache->ops->invalidate_object,
 985                                NULL, NULL);
 986         op->flags = FSCACHE_OP_ASYNC |
 987                 (1 << FSCACHE_OP_EXCLUSIVE) |
 988                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 989         trace_fscache_page_op(cookie, NULL, op, fscache_page_op_invalidate);
 990
 991         spin_lock(&cookie->lock);
 992         if (fscache_submit_exclusive_op(object, op) < 0)
 993                 goto submit_op_failed;
 994         spin_unlock(&cookie->lock);
 995         fscache_put_operation(op);
 996
 997         /* Once we've completed the invalidation, we know there will be no data
 998          * stored in the cache and thus we can reinstate the data-check-skip
 999          * optimisation.
1000          */
1001         set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
1002
1003         /* We can allow read and write requests to come in once again.  They'll
1004          * queue up behind our exclusive invalidation operation.
1005          */
1006         if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
1007                 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
1008         _leave(" [ok]");
1009         return transit_to(UPDATE_OBJECT);
1010
1011 nomem:
1012         fscache_mark_object_dead(object);
1013         fscache_unuse_cookie(object);
1014         _leave(" [ENOMEM]");
1015         return transit_to(KILL_OBJECT);
1016
1017 submit_op_failed:
1018         fscache_mark_object_dead(object);
1019         spin_unlock(&cookie->lock);
1020         fscache_unuse_cookie(object);
1021         kfree(op);
1022         _leave(" [EIO]");
1023         return transit_to(KILL_OBJECT);
1024 }
1025
1026 static const struct fscache_state *fscache_invalidate_object(struct fscache_object *object,
1027                                                              int event)
1028 {
1029         const struct fscache_state *s;
1030
1031         fscache_stat(&fscache_n_invalidates_run);
1032         fscache_stat(&fscache_n_cop_invalidate_object);
1033         s = _fscache_invalidate_object(object, event);
1034         fscache_stat_d(&fscache_n_cop_invalidate_object);
1035         return s;
1036 }
1037
1038 /*
1039  * Update auxiliary data.
1040  */
1041 static void fscache_update_aux_data(struct fscache_object *object)
1042 {
1043         fscache_stat(&fscache_n_updates_run);
1044         fscache_stat(&fscache_n_cop_update_object);
1045         object->cache->ops->update_object(object);
1046         fscache_stat_d(&fscache_n_cop_update_object);
1047 }
1048
1049 /*
1050  * Asynchronously update an object.
1051  */
1052 static const struct fscache_state *fscache_update_object(struct fscache_object *object,
1053                                                          int event)
1054 {
1055         _enter("{OBJ%x},%d", object->debug_id, event);
1056
1057         fscache_update_aux_data(object);
1058
1059         _leave("");
1060         return transit_to(WAIT_FOR_CMD);
1061 }
1062
1063 /**
1064  * fscache_object_retrying_stale - Note retrying stale object
1065  * @object: The object that will be retried
1066  *
1067  * Note that an object lookup found an on-disk object that was adjudged to be
1068  * stale and has been deleted.  The lookup will be retried.
1069  */
1070 void fscache_object_retrying_stale(struct fscache_object *object)
1071 {
1072         fscache_stat(&fscache_n_cache_no_space_reject);
1073 }
1074 EXPORT_SYMBOL(fscache_object_retrying_stale);
1075
1076 /**
1077  * fscache_object_mark_killed - Note that an object was killed
1078  * @object: The object that was culled
1079  * @why: The reason the object was killed.
1080  *
1081  * Note that an object was killed.  Returns true if the object was
1082  * already marked killed, false if it wasn't.
1083  */
1084 void fscache_object_mark_killed(struct fscache_object *object,
1085                                 enum fscache_why_object_killed why)
1086 {
1087         if (test_and_set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags)) {
1088                 pr_err("Error: Object already killed by cache [%s]\n",
1089                        object->cache->identifier);
1090                 return;
1091         }
1092
1093         switch (why) {
1094         case FSCACHE_OBJECT_NO_SPACE:
1095                 fscache_stat(&fscache_n_cache_no_space_reject);
1096                 break;
1097         case FSCACHE_OBJECT_IS_STALE:
1098                 fscache_stat(&fscache_n_cache_stale_objects);
1099                 break;
1100         case FSCACHE_OBJECT_WAS_RETIRED:
1101                 fscache_stat(&fscache_n_cache_retired_objects);
1102                 break;
1103         case FSCACHE_OBJECT_WAS_CULLED:
1104                 fscache_stat(&fscache_n_cache_culled_objects);
1105                 break;
1106         }
1107 }
1108 EXPORT_SYMBOL(fscache_object_mark_killed);
1109
1110 /*
1111  * The object is dead.  We can get here if an object gets queued by an event
1112  * that would lead to its death (such as EV_KILL) when the dispatcher is
1113  * already running (and so can be requeued) but hasn't yet cleared the event
1114  * mask.
1115  */
1116 static const struct fscache_state *fscache_object_dead(struct fscache_object *object,
1117                                                        int event)
1118 {
1119         if (!test_and_set_bit(FSCACHE_OBJECT_RUN_AFTER_DEAD,
1120                               &object->flags))
1121                 return NO_TRANSIT;
1122
1123         WARN(true, "FS-Cache object redispatched after death");
1124         return NO_TRANSIT;
1125 }