fs/bcachefs/journal_io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include "bcachefs.h"
   3 #include "alloc_background.h"
   4 #include "alloc_foreground.h"
   5 #include "btree_io.h"
   6 #include "btree_update_interior.h"
   7 #include "btree_write_buffer.h"
   8 #include "buckets.h"
   9 #include "checksum.h"
  10 #include "disk_groups.h"
  11 #include "error.h"
  12 #include "journal.h"
  13 #include "journal_io.h"
  14 #include "journal_reclaim.h"
  15 #include "journal_seq_blacklist.h"
  16 #include "replicas.h"
  17 #include "sb-clean.h"
  18 #include "trace.h"
  19
  20 void bch2_journal_ptrs_to_text(struct printbuf *out, struct bch_fs *c,
  21                                struct journal_replay *j)
  22 {
  23         darray_for_each(j->ptrs, i) {
  24                 struct bch_dev *ca = bch_dev_bkey_exists(c, i->dev);
  25                 u64 offset;
  26
  27                 div64_u64_rem(i->sector, ca->mi.bucket_size, &offset);
  28
  29                 if (i != j->ptrs.data)
  30                         prt_printf(out, " ");
  31                 prt_printf(out, "%u:%u:%u (sector %llu)",
  32                            i->dev, i->bucket, i->bucket_offset, i->sector);
  33         }
  34 }
  35
  36 static void bch2_journal_replay_to_text(struct printbuf *out, struct bch_fs *c,
  37                                         struct journal_replay *j)
  38 {
  39         prt_printf(out, "seq %llu ", le64_to_cpu(j->j.seq));
  40
  41         bch2_journal_ptrs_to_text(out, c, j);
  42
  43         for_each_jset_entry_type(entry, &j->j, BCH_JSET_ENTRY_datetime) {
  44                 struct jset_entry_datetime *datetime =
  45                         container_of(entry, struct jset_entry_datetime, entry);
  46                 bch2_prt_datetime(out, le64_to_cpu(datetime->seconds));
  47                 break;
  48         }
  49 }
  50
  51 static struct nonce journal_nonce(const struct jset *jset)
  52 {
  53         return (struct nonce) {{
  54                 [0] = 0,
  55                 [1] = ((__le32 *) &jset->seq)[0],
  56                 [2] = ((__le32 *) &jset->seq)[1],
  57                 [3] = BCH_NONCE_JOURNAL,
  58         }};
  59 }
  60
  61 static bool jset_csum_good(struct bch_fs *c, struct jset *j, struct bch_csum *csum)
  62 {
  63         if (!bch2_checksum_type_valid(c, JSET_CSUM_TYPE(j))) {
  64                 *csum = (struct bch_csum) {};
  65                 return false;
  66         }
  67
  68         *csum = csum_vstruct(c, JSET_CSUM_TYPE(j), journal_nonce(j), j);
  69         return !bch2_crc_cmp(j->csum, *csum);
  70 }
  71
  72 static inline u32 journal_entry_radix_idx(struct bch_fs *c, u64 seq)
  73 {
  74         return (seq - c->journal_entries_base_seq) & (~0U >> 1);
  75 }
  76
  77 static void __journal_replay_free(struct bch_fs *c,
  78                                   struct journal_replay *i)
  79 {
  80         struct journal_replay **p =
  81                 genradix_ptr(&c->journal_entries,
  82                              journal_entry_radix_idx(c, le64_to_cpu(i->j.seq)));
  83
  84         BUG_ON(*p != i);
  85         *p = NULL;
  86         kvfree(i);
  87 }
  88
  89 static void journal_replay_free(struct bch_fs *c, struct journal_replay *i, bool blacklisted)
  90 {
  91         if (blacklisted)
  92                 i->ignore_blacklisted = true;
  93         else
  94                 i->ignore_not_dirty = true;
  95
  96         if (!c->opts.read_entire_journal)
  97                 __journal_replay_free(c, i);
  98 }
  99
 100 struct journal_list {
 101         struct closure          cl;
 102         u64                     last_seq;
 103         struct mutex            lock;
 104         int                     ret;
 105 };
 106
 107 #define JOURNAL_ENTRY_ADD_OK            0
 108 #define JOURNAL_ENTRY_ADD_OUT_OF_RANGE  5
 109
 110 /*
 111  * Given a journal entry we just read, add it to the list of journal entries to
 112  * be replayed:
 113  */
 114 static int journal_entry_add(struct bch_fs *c, struct bch_dev *ca,
 115                              struct journal_ptr entry_ptr,
 116                              struct journal_list *jlist, struct jset *j)
 117 {
 118         struct genradix_iter iter;
 119         struct journal_replay **_i, *i, *dup;
 120         size_t bytes = vstruct_bytes(j);
 121         u64 last_seq = !JSET_NO_FLUSH(j) ? le64_to_cpu(j->last_seq) : 0;
 122         struct printbuf buf = PRINTBUF;
 123         int ret = JOURNAL_ENTRY_ADD_OK;
 124
 125         /* Is this entry older than the range we need? */
 126         if (!c->opts.read_entire_journal &&
 127             le64_to_cpu(j->seq) < jlist->last_seq)
 128                 return JOURNAL_ENTRY_ADD_OUT_OF_RANGE;
 129
 130         /*
 131          * genradixes are indexed by a ulong, not a u64, so we can't index them
 132          * by sequence number directly: Assume instead that they will all fall
 133          * within the range of +-2billion of the filrst one we find.
 134          */
 135         if (!c->journal_entries_base_seq)
 136                 c->journal_entries_base_seq = max_t(s64, 1, le64_to_cpu(j->seq) - S32_MAX);
 137
 138         /* Drop entries we don't need anymore */
 139         if (last_seq > jlist->last_seq && !c->opts.read_entire_journal) {
 140                 genradix_for_each_from(&c->journal_entries, iter, _i,
 141                                        journal_entry_radix_idx(c, jlist->last_seq)) {
 142                         i = *_i;
 143
 144                         if (journal_replay_ignore(i))
 145                                 continue;
 146
 147                         if (le64_to_cpu(i->j.seq) >= last_seq)
 148                                 break;
 149
 150                         journal_replay_free(c, i, false);
 151                 }
 152         }
 153
 154         jlist->last_seq = max(jlist->last_seq, last_seq);
 155
 156         _i = genradix_ptr_alloc(&c->journal_entries,
 157                                 journal_entry_radix_idx(c, le64_to_cpu(j->seq)),
 158                                 GFP_KERNEL);
 159         if (!_i)
 160                 return -BCH_ERR_ENOMEM_journal_entry_add;
 161
 162         /*
 163          * Duplicate journal entries? If so we want the one that didn't have a
 164          * checksum error:
 165          */
 166         dup = *_i;
 167         if (dup) {
 168                 bool identical = bytes == vstruct_bytes(&dup->j) &&
 169                         !memcmp(j, &dup->j, bytes);
 170                 bool not_identical = !identical &&
 171                         entry_ptr.csum_good &&
 172                         dup->csum_good;
 173
 174                 bool same_device = false;
 175                 darray_for_each(dup->ptrs, ptr)
 176                         if (ptr->dev == ca->dev_idx)
 177                                 same_device = true;
 178
 179                 ret = darray_push(&dup->ptrs, entry_ptr);
 180                 if (ret)
 181                         goto out;
 182
 183                 bch2_journal_replay_to_text(&buf, c, dup);
 184
 185                 fsck_err_on(same_device,
 186                             c, journal_entry_dup_same_device,
 187                             "duplicate journal entry on same device\n  %s",
 188                             buf.buf);
 189
 190                 fsck_err_on(not_identical,
 191                             c, journal_entry_replicas_data_mismatch,
 192                             "found duplicate but non identical journal entries\n  %s",
 193                             buf.buf);
 194
 195                 if (entry_ptr.csum_good && !identical)
 196                         goto replace;
 197
 198                 goto out;
 199         }
 200 replace:
 201         i = kvmalloc(offsetof(struct journal_replay, j) + bytes, GFP_KERNEL);
 202         if (!i)
 203                 return -BCH_ERR_ENOMEM_journal_entry_add;
 204
 205         darray_init(&i->ptrs);
 206         i->csum_good            = entry_ptr.csum_good;
 207         i->ignore_blacklisted   = false;
 208         i->ignore_not_dirty     = false;
 209         unsafe_memcpy(&i->j, j, bytes, "embedded variable length struct");
 210
 211         if (dup) {
 212                 /* The first ptr should represent the jset we kept: */
 213                 darray_for_each(dup->ptrs, ptr)
 214                         darray_push(&i->ptrs, *ptr);
 215                 __journal_replay_free(c, dup);
 216         } else {
 217                 darray_push(&i->ptrs, entry_ptr);
 218         }
 219
 220         *_i = i;
 221 out:
 222 fsck_err:
 223         printbuf_exit(&buf);
 224         return ret;
 225 }
 226
 227 /* this fills in a range with empty jset_entries: */
 228 static void journal_entry_null_range(void *start, void *end)
 229 {
 230         struct jset_entry *entry;
 231
 232         for (entry = start; entry != end; entry = vstruct_next(entry))
 233                 memset(entry, 0, sizeof(*entry));
 234 }
 235
 236 #define JOURNAL_ENTRY_REREAD    5
 237 #define JOURNAL_ENTRY_NONE      6
 238 #define JOURNAL_ENTRY_BAD       7
 239
 240 static void journal_entry_err_msg(struct printbuf *out,
 241                                   u32 version,
 242                                   struct jset *jset,
 243                                   struct jset_entry *entry)
 244 {
 245         prt_str(out, "invalid journal entry, version=");
 246         bch2_version_to_text(out, version);
 247
 248         if (entry) {
 249                 prt_str(out, " type=");
 250                 bch2_prt_jset_entry_type(out, entry->type);
 251         }
 252
 253         if (!jset) {
 254                 prt_printf(out, " in superblock");
 255         } else {
 256
 257                 prt_printf(out, " seq=%llu", le64_to_cpu(jset->seq));
 258
 259                 if (entry)
 260                         prt_printf(out, " offset=%zi/%u",
 261                                    (u64 *) entry - jset->_data,
 262                                    le32_to_cpu(jset->u64s));
 263         }
 264
 265         prt_str(out, ": ");
 266 }
 267
 268 #define journal_entry_err(c, version, jset, entry, _err, msg, ...)      \
 269 ({                                                                      \
 270         struct printbuf _buf = PRINTBUF;                                \
 271                                                                         \
 272         journal_entry_err_msg(&_buf, version, jset, entry);             \
 273         prt_printf(&_buf, msg, ##__VA_ARGS__);                          \
 274                                                                         \
 275         switch (flags & BKEY_INVALID_WRITE) {                           \
 276         case READ:                                                      \
 277                 mustfix_fsck_err(c, _err, "%s", _buf.buf);              \
 278                 break;                                                  \
 279         case WRITE:                                                     \
 280                 bch2_sb_error_count(c, BCH_FSCK_ERR_##_err);            \
 281                 bch_err(c, "corrupt metadata before write: %s\n", _buf.buf);\
 282                 if (bch2_fs_inconsistent(c)) {                          \
 283                         ret = -BCH_ERR_fsck_errors_not_fixed;           \
 284                         goto fsck_err;                                  \
 285                 }                                                       \
 286                 break;                                                  \
 287         }                                                               \
 288                                                                         \
 289         printbuf_exit(&_buf);                                           \
 290         true;                                                           \
 291 })
 292
 293 #define journal_entry_err_on(cond, ...)                                 \
 294         ((cond) ? journal_entry_err(__VA_ARGS__) : false)
 295
 296 #define FSCK_DELETED_KEY        5
 297
 298 static int journal_validate_key(struct bch_fs *c,
 299                                 struct jset *jset,
 300                                 struct jset_entry *entry,
 301                                 unsigned level, enum btree_id btree_id,
 302                                 struct bkey_i *k,
 303                                 unsigned version, int big_endian,
 304                                 enum bkey_invalid_flags flags)
 305 {
 306         int write = flags & BKEY_INVALID_WRITE;
 307         void *next = vstruct_next(entry);
 308         struct printbuf buf = PRINTBUF;
 309         int ret = 0;
 310
 311         if (journal_entry_err_on(!k->k.u64s,
 312                                  c, version, jset, entry,
 313                                  journal_entry_bkey_u64s_0,
 314                                  "k->u64s 0")) {
 315                 entry->u64s = cpu_to_le16((u64 *) k - entry->_data);
 316                 journal_entry_null_range(vstruct_next(entry), next);
 317                 return FSCK_DELETED_KEY;
 318         }
 319
 320         if (journal_entry_err_on((void *) bkey_next(k) >
 321                                  (void *) vstruct_next(entry),
 322                                  c, version, jset, entry,
 323                                  journal_entry_bkey_past_end,
 324                                  "extends past end of journal entry")) {
 325                 entry->u64s = cpu_to_le16((u64 *) k - entry->_data);
 326                 journal_entry_null_range(vstruct_next(entry), next);
 327                 return FSCK_DELETED_KEY;
 328         }
 329
 330         if (journal_entry_err_on(k->k.format != KEY_FORMAT_CURRENT,
 331                                  c, version, jset, entry,
 332                                  journal_entry_bkey_bad_format,
 333                                  "bad format %u", k->k.format)) {
 334                 le16_add_cpu(&entry->u64s, -((u16) k->k.u64s));
 335                 memmove(k, bkey_next(k), next - (void *) bkey_next(k));
 336                 journal_entry_null_range(vstruct_next(entry), next);
 337                 return FSCK_DELETED_KEY;
 338         }
 339
 340         if (!write)
 341                 bch2_bkey_compat(level, btree_id, version, big_endian,
 342                                  write, NULL, bkey_to_packed(k));
 343
 344         if (bch2_bkey_invalid(c, bkey_i_to_s_c(k),
 345                               __btree_node_type(level, btree_id), write, &buf)) {
 346                 printbuf_reset(&buf);
 347                 journal_entry_err_msg(&buf, version, jset, entry);
 348                 prt_newline(&buf);
 349                 printbuf_indent_add(&buf, 2);
 350
 351                 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(k));
 352                 prt_newline(&buf);
 353                 bch2_bkey_invalid(c, bkey_i_to_s_c(k),
 354                                   __btree_node_type(level, btree_id), write, &buf);
 355
 356                 mustfix_fsck_err(c, journal_entry_bkey_invalid,
 357                                  "%s", buf.buf);
 358
 359                 le16_add_cpu(&entry->u64s, -((u16) k->k.u64s));
 360                 memmove(k, bkey_next(k), next - (void *) bkey_next(k));
 361                 journal_entry_null_range(vstruct_next(entry), next);
 362
 363                 printbuf_exit(&buf);
 364                 return FSCK_DELETED_KEY;
 365         }
 366
 367         if (write)
 368                 bch2_bkey_compat(level, btree_id, version, big_endian,
 369                                  write, NULL, bkey_to_packed(k));
 370 fsck_err:
 371         printbuf_exit(&buf);
 372         return ret;
 373 }
 374
 375 static int journal_entry_btree_keys_validate(struct bch_fs *c,
 376                                 struct jset *jset,
 377                                 struct jset_entry *entry,
 378                                 unsigned version, int big_endian,
 379                                 enum bkey_invalid_flags flags)
 380 {
 381         struct bkey_i *k = entry->start;
 382
 383         while (k != vstruct_last(entry)) {
 384                 int ret = journal_validate_key(c, jset, entry,
 385                                                entry->level,
 386                                                entry->btree_id,
 387                                                k, version, big_endian,
 388                                                flags|BKEY_INVALID_JOURNAL);
 389                 if (ret == FSCK_DELETED_KEY)
 390                         continue;
 391
 392                 k = bkey_next(k);
 393         }
 394
 395         return 0;
 396 }
 397
 398 static void journal_entry_btree_keys_to_text(struct printbuf *out, struct bch_fs *c,
 399                                              struct jset_entry *entry)
 400 {
 401         bool first = true;
 402
 403         jset_entry_for_each_key(entry, k) {
 404                 if (!first) {
 405                         prt_newline(out);
 406                         bch2_prt_jset_entry_type(out, entry->type);
 407                         prt_str(out, ": ");
 408                 }
 409                 prt_printf(out, "btree=%s l=%u ", bch2_btree_id_str(entry->btree_id), entry->level);
 410                 bch2_bkey_val_to_text(out, c, bkey_i_to_s_c(k));
 411                 first = false;
 412         }
 413 }
 414
 415 static int journal_entry_btree_root_validate(struct bch_fs *c,
 416                                 struct jset *jset,
 417                                 struct jset_entry *entry,
 418                                 unsigned version, int big_endian,
 419                                 enum bkey_invalid_flags flags)
 420 {
 421         struct bkey_i *k = entry->start;
 422         int ret = 0;
 423
 424         if (journal_entry_err_on(!entry->u64s ||
 425                                  le16_to_cpu(entry->u64s) != k->k.u64s,
 426                                  c, version, jset, entry,
 427                                  journal_entry_btree_root_bad_size,
 428                                  "invalid btree root journal entry: wrong number of keys")) {
 429                 void *next = vstruct_next(entry);
 430                 /*
 431                  * we don't want to null out this jset_entry,
 432                  * just the contents, so that later we can tell
 433                  * we were _supposed_ to have a btree root
 434                  */
 435                 entry->u64s = 0;
 436                 journal_entry_null_range(vstruct_next(entry), next);
 437                 return 0;
 438         }
 439
 440         ret = journal_validate_key(c, jset, entry, 1, entry->btree_id, k,
 441                                    version, big_endian, flags);
 442         if (ret == FSCK_DELETED_KEY)
 443                 ret = 0;
 444 fsck_err:
 445         return ret;
 446 }
 447
 448 static void journal_entry_btree_root_to_text(struct printbuf *out, struct bch_fs *c,
 449                                              struct jset_entry *entry)
 450 {
 451         journal_entry_btree_keys_to_text(out, c, entry);
 452 }
 453
 454 static int journal_entry_prio_ptrs_validate(struct bch_fs *c,
 455                                 struct jset *jset,
 456                                 struct jset_entry *entry,
 457                                 unsigned version, int big_endian,
 458                                 enum bkey_invalid_flags flags)
 459 {
 460         /* obsolete, don't care: */
 461         return 0;
 462 }
 463
 464 static void journal_entry_prio_ptrs_to_text(struct printbuf *out, struct bch_fs *c,
 465                                             struct jset_entry *entry)
 466 {
 467 }
 468
 469 static int journal_entry_blacklist_validate(struct bch_fs *c,
 470                                 struct jset *jset,
 471                                 struct jset_entry *entry,
 472                                 unsigned version, int big_endian,
 473                                 enum bkey_invalid_flags flags)
 474 {
 475         int ret = 0;
 476
 477         if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 1,
 478                                  c, version, jset, entry,
 479                                  journal_entry_blacklist_bad_size,
 480                 "invalid journal seq blacklist entry: bad size")) {
 481                 journal_entry_null_range(entry, vstruct_next(entry));
 482         }
 483 fsck_err:
 484         return ret;
 485 }
 486
 487 static void journal_entry_blacklist_to_text(struct printbuf *out, struct bch_fs *c,
 488                                             struct jset_entry *entry)
 489 {
 490         struct jset_entry_blacklist *bl =
 491                 container_of(entry, struct jset_entry_blacklist, entry);
 492
 493         prt_printf(out, "seq=%llu", le64_to_cpu(bl->seq));
 494 }
 495
 496 static int journal_entry_blacklist_v2_validate(struct bch_fs *c,
 497                                 struct jset *jset,
 498                                 struct jset_entry *entry,
 499                                 unsigned version, int big_endian,
 500                                 enum bkey_invalid_flags flags)
 501 {
 502         struct jset_entry_blacklist_v2 *bl_entry;
 503         int ret = 0;
 504
 505         if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 2,
 506                                  c, version, jset, entry,
 507                                  journal_entry_blacklist_v2_bad_size,
 508                 "invalid journal seq blacklist entry: bad size")) {
 509                 journal_entry_null_range(entry, vstruct_next(entry));
 510                 goto out;
 511         }
 512
 513         bl_entry = container_of(entry, struct jset_entry_blacklist_v2, entry);
 514
 515         if (journal_entry_err_on(le64_to_cpu(bl_entry->start) >
 516                                  le64_to_cpu(bl_entry->end),
 517                                  c, version, jset, entry,
 518                                  journal_entry_blacklist_v2_start_past_end,
 519                 "invalid journal seq blacklist entry: start > end")) {
 520                 journal_entry_null_range(entry, vstruct_next(entry));
 521         }
 522 out:
 523 fsck_err:
 524         return ret;
 525 }
 526
 527 static void journal_entry_blacklist_v2_to_text(struct printbuf *out, struct bch_fs *c,
 528                                                struct jset_entry *entry)
 529 {
 530         struct jset_entry_blacklist_v2 *bl =
 531                 container_of(entry, struct jset_entry_blacklist_v2, entry);
 532
 533         prt_printf(out, "start=%llu end=%llu",
 534                le64_to_cpu(bl->start),
 535                le64_to_cpu(bl->end));
 536 }
 537
 538 static int journal_entry_usage_validate(struct bch_fs *c,
 539                                 struct jset *jset,
 540                                 struct jset_entry *entry,
 541                                 unsigned version, int big_endian,
 542                                 enum bkey_invalid_flags flags)
 543 {
 544         struct jset_entry_usage *u =
 545                 container_of(entry, struct jset_entry_usage, entry);
 546         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 547         int ret = 0;
 548
 549         if (journal_entry_err_on(bytes < sizeof(*u),
 550                                  c, version, jset, entry,
 551                                  journal_entry_usage_bad_size,
 552                                  "invalid journal entry usage: bad size")) {
 553                 journal_entry_null_range(entry, vstruct_next(entry));
 554                 return ret;
 555         }
 556
 557 fsck_err:
 558         return ret;
 559 }
 560
 561 static void journal_entry_usage_to_text(struct printbuf *out, struct bch_fs *c,
 562                                         struct jset_entry *entry)
 563 {
 564         struct jset_entry_usage *u =
 565                 container_of(entry, struct jset_entry_usage, entry);
 566
 567         prt_str(out, "type=");
 568         bch2_prt_fs_usage_type(out, u->entry.btree_id);
 569         prt_printf(out, " v=%llu", le64_to_cpu(u->v));
 570 }
 571
 572 static int journal_entry_data_usage_validate(struct bch_fs *c,
 573                                 struct jset *jset,
 574                                 struct jset_entry *entry,
 575                                 unsigned version, int big_endian,
 576                                 enum bkey_invalid_flags flags)
 577 {
 578         struct jset_entry_data_usage *u =
 579                 container_of(entry, struct jset_entry_data_usage, entry);
 580         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 581         struct printbuf err = PRINTBUF;
 582         int ret = 0;
 583
 584         if (journal_entry_err_on(bytes < sizeof(*u) ||
 585                                  bytes < sizeof(*u) + u->r.nr_devs,
 586                                  c, version, jset, entry,
 587                                  journal_entry_data_usage_bad_size,
 588                                  "invalid journal entry usage: bad size")) {
 589                 journal_entry_null_range(entry, vstruct_next(entry));
 590                 goto out;
 591         }
 592
 593         if (journal_entry_err_on(bch2_replicas_entry_validate(&u->r, c->disk_sb.sb, &err),
 594                                  c, version, jset, entry,
 595                                  journal_entry_data_usage_bad_size,
 596                                  "invalid journal entry usage: %s", err.buf)) {
 597                 journal_entry_null_range(entry, vstruct_next(entry));
 598                 goto out;
 599         }
 600 out:
 601 fsck_err:
 602         printbuf_exit(&err);
 603         return ret;
 604 }
 605
 606 static void journal_entry_data_usage_to_text(struct printbuf *out, struct bch_fs *c,
 607                                              struct jset_entry *entry)
 608 {
 609         struct jset_entry_data_usage *u =
 610                 container_of(entry, struct jset_entry_data_usage, entry);
 611
 612         bch2_replicas_entry_to_text(out, &u->r);
 613         prt_printf(out, "=%llu", le64_to_cpu(u->v));
 614 }
 615
 616 static int journal_entry_clock_validate(struct bch_fs *c,
 617                                 struct jset *jset,
 618                                 struct jset_entry *entry,
 619                                 unsigned version, int big_endian,
 620                                 enum bkey_invalid_flags flags)
 621 {
 622         struct jset_entry_clock *clock =
 623                 container_of(entry, struct jset_entry_clock, entry);
 624         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 625         int ret = 0;
 626
 627         if (journal_entry_err_on(bytes != sizeof(*clock),
 628                                  c, version, jset, entry,
 629                                  journal_entry_clock_bad_size,
 630                                  "bad size")) {
 631                 journal_entry_null_range(entry, vstruct_next(entry));
 632                 return ret;
 633         }
 634
 635         if (journal_entry_err_on(clock->rw > 1,
 636                                  c, version, jset, entry,
 637                                  journal_entry_clock_bad_rw,
 638                                  "bad rw")) {
 639                 journal_entry_null_range(entry, vstruct_next(entry));
 640                 return ret;
 641         }
 642
 643 fsck_err:
 644         return ret;
 645 }
 646
 647 static void journal_entry_clock_to_text(struct printbuf *out, struct bch_fs *c,
 648                                         struct jset_entry *entry)
 649 {
 650         struct jset_entry_clock *clock =
 651                 container_of(entry, struct jset_entry_clock, entry);
 652
 653         prt_printf(out, "%s=%llu", clock->rw ? "write" : "read", le64_to_cpu(clock->time));
 654 }
 655
 656 static int journal_entry_dev_usage_validate(struct bch_fs *c,
 657                                 struct jset *jset,
 658                                 struct jset_entry *entry,
 659                                 unsigned version, int big_endian,
 660                                 enum bkey_invalid_flags flags)
 661 {
 662         struct jset_entry_dev_usage *u =
 663                 container_of(entry, struct jset_entry_dev_usage, entry);
 664         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 665         unsigned expected = sizeof(*u);
 666         unsigned dev;
 667         int ret = 0;
 668
 669         if (journal_entry_err_on(bytes < expected,
 670                                  c, version, jset, entry,
 671                                  journal_entry_dev_usage_bad_size,
 672                                  "bad size (%u < %u)",
 673                                  bytes, expected)) {
 674                 journal_entry_null_range(entry, vstruct_next(entry));
 675                 return ret;
 676         }
 677
 678         dev = le32_to_cpu(u->dev);
 679
 680         if (journal_entry_err_on(!bch2_dev_exists2(c, dev),
 681                                  c, version, jset, entry,
 682                                  journal_entry_dev_usage_bad_dev,
 683                                  "bad dev")) {
 684                 journal_entry_null_range(entry, vstruct_next(entry));
 685                 return ret;
 686         }
 687
 688         if (journal_entry_err_on(u->pad,
 689                                  c, version, jset, entry,
 690                                  journal_entry_dev_usage_bad_pad,
 691                                  "bad pad")) {
 692                 journal_entry_null_range(entry, vstruct_next(entry));
 693                 return ret;
 694         }
 695
 696 fsck_err:
 697         return ret;
 698 }
 699
 700 static void journal_entry_dev_usage_to_text(struct printbuf *out, struct bch_fs *c,
 701                                             struct jset_entry *entry)
 702 {
 703         struct jset_entry_dev_usage *u =
 704                 container_of(entry, struct jset_entry_dev_usage, entry);
 705         unsigned i, nr_types = jset_entry_dev_usage_nr_types(u);
 706
 707         prt_printf(out, "dev=%u", le32_to_cpu(u->dev));
 708
 709         for (i = 0; i < nr_types; i++) {
 710                 bch2_prt_data_type(out, i);
 711                 prt_printf(out, ": buckets=%llu sectors=%llu fragmented=%llu",
 712                        le64_to_cpu(u->d[i].buckets),
 713                        le64_to_cpu(u->d[i].sectors),
 714                        le64_to_cpu(u->d[i].fragmented));
 715         }
 716 }
 717
 718 static int journal_entry_log_validate(struct bch_fs *c,
 719                                 struct jset *jset,
 720                                 struct jset_entry *entry,
 721                                 unsigned version, int big_endian,
 722                                 enum bkey_invalid_flags flags)
 723 {
 724         return 0;
 725 }
 726
 727 static void journal_entry_log_to_text(struct printbuf *out, struct bch_fs *c,
 728                                       struct jset_entry *entry)
 729 {
 730         struct jset_entry_log *l = container_of(entry, struct jset_entry_log, entry);
 731         unsigned bytes = vstruct_bytes(entry) - offsetof(struct jset_entry_log, d);
 732
 733         prt_printf(out, "%.*s", bytes, l->d);
 734 }
 735
 736 static int journal_entry_overwrite_validate(struct bch_fs *c,
 737                                 struct jset *jset,
 738                                 struct jset_entry *entry,
 739                                 unsigned version, int big_endian,
 740                                 enum bkey_invalid_flags flags)
 741 {
 742         return journal_entry_btree_keys_validate(c, jset, entry,
 743                                 version, big_endian, READ);
 744 }
 745
 746 static void journal_entry_overwrite_to_text(struct printbuf *out, struct bch_fs *c,
 747                                             struct jset_entry *entry)
 748 {
 749         journal_entry_btree_keys_to_text(out, c, entry);
 750 }
 751
 752 static int journal_entry_write_buffer_keys_validate(struct bch_fs *c,
 753                                 struct jset *jset,
 754                                 struct jset_entry *entry,
 755                                 unsigned version, int big_endian,
 756                                 enum bkey_invalid_flags flags)
 757 {
 758         return journal_entry_btree_keys_validate(c, jset, entry,
 759                                 version, big_endian, READ);
 760 }
 761
 762 static void journal_entry_write_buffer_keys_to_text(struct printbuf *out, struct bch_fs *c,
 763                                             struct jset_entry *entry)
 764 {
 765         journal_entry_btree_keys_to_text(out, c, entry);
 766 }
 767
 768 static int journal_entry_datetime_validate(struct bch_fs *c,
 769                                 struct jset *jset,
 770                                 struct jset_entry *entry,
 771                                 unsigned version, int big_endian,
 772                                 enum bkey_invalid_flags flags)
 773 {
 774         unsigned bytes = vstruct_bytes(entry);
 775         unsigned expected = 16;
 776         int ret = 0;
 777
 778         if (journal_entry_err_on(vstruct_bytes(entry) < expected,
 779                                  c, version, jset, entry,
 780                                  journal_entry_dev_usage_bad_size,
 781                                  "bad size (%u < %u)",
 782                                  bytes, expected)) {
 783                 journal_entry_null_range(entry, vstruct_next(entry));
 784                 return ret;
 785         }
 786 fsck_err:
 787         return ret;
 788 }
 789
 790 static void journal_entry_datetime_to_text(struct printbuf *out, struct bch_fs *c,
 791                                             struct jset_entry *entry)
 792 {
 793         struct jset_entry_datetime *datetime =
 794                 container_of(entry, struct jset_entry_datetime, entry);
 795
 796         bch2_prt_datetime(out, le64_to_cpu(datetime->seconds));
 797 }
 798
 799 struct jset_entry_ops {
 800         int (*validate)(struct bch_fs *, struct jset *,
 801                         struct jset_entry *, unsigned, int,
 802                         enum bkey_invalid_flags);
 803         void (*to_text)(struct printbuf *, struct bch_fs *, struct jset_entry *);
 804 };
 805
 806 static const struct jset_entry_ops bch2_jset_entry_ops[] = {
 807 #define x(f, nr)                                                \
 808         [BCH_JSET_ENTRY_##f]    = (struct jset_entry_ops) {     \
 809                 .validate       = journal_entry_##f##_validate, \
 810                 .to_text        = journal_entry_##f##_to_text,  \
 811         },
 812         BCH_JSET_ENTRY_TYPES()
 813 #undef x
 814 };
 815
 816 int bch2_journal_entry_validate(struct bch_fs *c,
 817                                 struct jset *jset,
 818                                 struct jset_entry *entry,
 819                                 unsigned version, int big_endian,
 820                                 enum bkey_invalid_flags flags)
 821 {
 822         return entry->type < BCH_JSET_ENTRY_NR
 823                 ? bch2_jset_entry_ops[entry->type].validate(c, jset, entry,
 824                                 version, big_endian, flags)
 825                 : 0;
 826 }
 827
 828 void bch2_journal_entry_to_text(struct printbuf *out, struct bch_fs *c,
 829                                 struct jset_entry *entry)
 830 {
 831         bch2_prt_jset_entry_type(out, entry->type);
 832
 833         if (entry->type < BCH_JSET_ENTRY_NR) {
 834                 prt_str(out, ": ");
 835                 bch2_jset_entry_ops[entry->type].to_text(out, c, entry);
 836         }
 837 }
 838
 839 static int jset_validate_entries(struct bch_fs *c, struct jset *jset,
 840                                  enum bkey_invalid_flags flags)
 841 {
 842         unsigned version = le32_to_cpu(jset->version);
 843         int ret = 0;
 844
 845         vstruct_for_each(jset, entry) {
 846                 if (journal_entry_err_on(vstruct_next(entry) > vstruct_last(jset),
 847                                 c, version, jset, entry,
 848                                 journal_entry_past_jset_end,
 849                                 "journal entry extends past end of jset")) {
 850                         jset->u64s = cpu_to_le32((u64 *) entry - jset->_data);
 851                         break;
 852                 }
 853
 854                 ret = bch2_journal_entry_validate(c, jset, entry,
 855                                         version, JSET_BIG_ENDIAN(jset), flags);
 856                 if (ret)
 857                         break;
 858         }
 859 fsck_err:
 860         return ret;
 861 }
 862
 863 static int jset_validate(struct bch_fs *c,
 864                          struct bch_dev *ca,
 865                          struct jset *jset, u64 sector,
 866                          enum bkey_invalid_flags flags)
 867 {
 868         unsigned version;
 869         int ret = 0;
 870
 871         if (le64_to_cpu(jset->magic) != jset_magic(c))
 872                 return JOURNAL_ENTRY_NONE;
 873
 874         version = le32_to_cpu(jset->version);
 875         if (journal_entry_err_on(!bch2_version_compatible(version),
 876                         c, version, jset, NULL,
 877                         jset_unsupported_version,
 878                         "%s sector %llu seq %llu: incompatible journal entry version %u.%u",
 879                         ca ? ca->name : c->name,
 880                         sector, le64_to_cpu(jset->seq),
 881                         BCH_VERSION_MAJOR(version),
 882                         BCH_VERSION_MINOR(version))) {
 883                 /* don't try to continue: */
 884                 return -EINVAL;
 885         }
 886
 887         if (journal_entry_err_on(!bch2_checksum_type_valid(c, JSET_CSUM_TYPE(jset)),
 888                         c, version, jset, NULL,
 889                         jset_unknown_csum,
 890                         "%s sector %llu seq %llu: journal entry with unknown csum type %llu",
 891                         ca ? ca->name : c->name,
 892                         sector, le64_to_cpu(jset->seq),
 893                         JSET_CSUM_TYPE(jset)))
 894                 ret = JOURNAL_ENTRY_BAD;
 895
 896         /* last_seq is ignored when JSET_NO_FLUSH is true */
 897         if (journal_entry_err_on(!JSET_NO_FLUSH(jset) &&
 898                                  le64_to_cpu(jset->last_seq) > le64_to_cpu(jset->seq),
 899                                  c, version, jset, NULL,
 900                                  jset_last_seq_newer_than_seq,
 901                                  "invalid journal entry: last_seq > seq (%llu > %llu)",
 902                                  le64_to_cpu(jset->last_seq),
 903                                  le64_to_cpu(jset->seq))) {
 904                 jset->last_seq = jset->seq;
 905                 return JOURNAL_ENTRY_BAD;
 906         }
 907
 908         ret = jset_validate_entries(c, jset, flags);
 909 fsck_err:
 910         return ret;
 911 }
 912
 913 static int jset_validate_early(struct bch_fs *c,
 914                          struct bch_dev *ca,
 915                          struct jset *jset, u64 sector,
 916                          unsigned bucket_sectors_left,
 917                          unsigned sectors_read)
 918 {
 919         size_t bytes = vstruct_bytes(jset);
 920         unsigned version;
 921         enum bkey_invalid_flags flags = BKEY_INVALID_JOURNAL;
 922         int ret = 0;
 923
 924         if (le64_to_cpu(jset->magic) != jset_magic(c))
 925                 return JOURNAL_ENTRY_NONE;
 926
 927         version = le32_to_cpu(jset->version);
 928         if (journal_entry_err_on(!bch2_version_compatible(version),
 929                         c, version, jset, NULL,
 930                         jset_unsupported_version,
 931                         "%s sector %llu seq %llu: unknown journal entry version %u.%u",
 932                         ca ? ca->name : c->name,
 933                         sector, le64_to_cpu(jset->seq),
 934                         BCH_VERSION_MAJOR(version),
 935                         BCH_VERSION_MINOR(version))) {
 936                 /* don't try to continue: */
 937                 return -EINVAL;
 938         }
 939
 940         if (bytes > (sectors_read << 9) &&
 941             sectors_read < bucket_sectors_left)
 942                 return JOURNAL_ENTRY_REREAD;
 943
 944         if (journal_entry_err_on(bytes > bucket_sectors_left << 9,
 945                         c, version, jset, NULL,
 946                         jset_past_bucket_end,
 947                         "%s sector %llu seq %llu: journal entry too big (%zu bytes)",
 948                         ca ? ca->name : c->name,
 949                         sector, le64_to_cpu(jset->seq), bytes))
 950                 le32_add_cpu(&jset->u64s,
 951                              -((bytes - (bucket_sectors_left << 9)) / 8));
 952 fsck_err:
 953         return ret;
 954 }
 955
 956 struct journal_read_buf {
 957         void            *data;
 958         size_t          size;
 959 };
 960
 961 static int journal_read_buf_realloc(struct journal_read_buf *b,
 962                                     size_t new_size)
 963 {
 964         void *n;
 965
 966         /* the bios are sized for this many pages, max: */
 967         if (new_size > JOURNAL_ENTRY_SIZE_MAX)
 968                 return -BCH_ERR_ENOMEM_journal_read_buf_realloc;
 969
 970         new_size = roundup_pow_of_two(new_size);
 971         n = kvmalloc(new_size, GFP_KERNEL);
 972         if (!n)
 973                 return -BCH_ERR_ENOMEM_journal_read_buf_realloc;
 974
 975         kvfree(b->data);
 976         b->data = n;
 977         b->size = new_size;
 978         return 0;
 979 }
 980
 981 static int journal_read_bucket(struct bch_dev *ca,
 982                                struct journal_read_buf *buf,
 983                                struct journal_list *jlist,
 984                                unsigned bucket)
 985 {
 986         struct bch_fs *c = ca->fs;
 987         struct journal_device *ja = &ca->journal;
 988         struct jset *j = NULL;
 989         unsigned sectors, sectors_read = 0;
 990         u64 offset = bucket_to_sector(ca, ja->buckets[bucket]),
 991             end = offset + ca->mi.bucket_size;
 992         bool saw_bad = false, csum_good;
 993         struct printbuf err = PRINTBUF;
 994         int ret = 0;
 995
 996         pr_debug("reading %u", bucket);
 997
 998         while (offset < end) {
 999                 if (!sectors_read) {
1000                         struct bio *bio;
1001                         unsigned nr_bvecs;
1002 reread:
1003                         sectors_read = min_t(unsigned,
1004                                 end - offset, buf->size >> 9);
1005                         nr_bvecs = buf_pages(buf->data, sectors_read << 9);
1006
1007                         bio = bio_kmalloc(nr_bvecs, GFP_KERNEL);
1008                         bio_init(bio, ca->disk_sb.bdev, bio->bi_inline_vecs, nr_bvecs, REQ_OP_READ);
1009
1010                         bio->bi_iter.bi_sector = offset;
1011                         bch2_bio_map(bio, buf->data, sectors_read << 9);
1012
1013                         ret = submit_bio_wait(bio);
1014                         kfree(bio);
1015
1016                         if (bch2_dev_io_err_on(ret, ca, BCH_MEMBER_ERROR_read,
1017                                                "journal read error: sector %llu",
1018                                                offset) ||
1019                             bch2_meta_read_fault("journal")) {
1020                                 /*
1021                                  * We don't error out of the recovery process
1022                                  * here, since the relevant journal entry may be
1023                                  * found on a different device, and missing or
1024                                  * no journal entries will be handled later
1025                                  */
1026                                 goto out;
1027                         }
1028
1029                         j = buf->data;
1030                 }
1031
1032                 ret = jset_validate_early(c, ca, j, offset,
1033                                     end - offset, sectors_read);
1034                 switch (ret) {
1035                 case 0:
1036                         sectors = vstruct_sectors(j, c->block_bits);
1037                         break;
1038                 case JOURNAL_ENTRY_REREAD:
1039                         if (vstruct_bytes(j) > buf->size) {
1040                                 ret = journal_read_buf_realloc(buf,
1041                                                         vstruct_bytes(j));
1042                                 if (ret)
1043                                         goto err;
1044                         }
1045                         goto reread;
1046                 case JOURNAL_ENTRY_NONE:
1047                         if (!saw_bad)
1048                                 goto out;
1049                         /*
1050                          * On checksum error we don't really trust the size
1051                          * field of the journal entry we read, so try reading
1052                          * again at next block boundary:
1053                          */
1054                         sectors = block_sectors(c);
1055                         goto next_block;
1056                 default:
1057                         goto err;
1058                 }
1059
1060                 /*
1061                  * This happens sometimes if we don't have discards on -
1062                  * when we've partially overwritten a bucket with new
1063                  * journal entries. We don't need the rest of the
1064                  * bucket:
1065                  */
1066                 if (le64_to_cpu(j->seq) < ja->bucket_seq[bucket])
1067                         goto out;
1068
1069                 ja->bucket_seq[bucket] = le64_to_cpu(j->seq);
1070
1071                 enum bch_csum_type csum_type = JSET_CSUM_TYPE(j);
1072                 struct bch_csum csum;
1073                 csum_good = jset_csum_good(c, j, &csum);
1074
1075                 if (bch2_dev_io_err_on(!csum_good, ca, BCH_MEMBER_ERROR_checksum,
1076                                        "%s",
1077                                        (printbuf_reset(&err),
1078                                         prt_str(&err, "journal "),
1079                                         bch2_csum_err_msg(&err, csum_type, j->csum, csum),
1080                                         err.buf)))
1081                         saw_bad = true;
1082
1083                 ret = bch2_encrypt(c, JSET_CSUM_TYPE(j), journal_nonce(j),
1084                              j->encrypted_start,
1085                              vstruct_end(j) - (void *) j->encrypted_start);
1086                 bch2_fs_fatal_err_on(ret, c, "decrypting journal entry: %s", bch2_err_str(ret));
1087
1088                 mutex_lock(&jlist->lock);
1089                 ret = journal_entry_add(c, ca, (struct journal_ptr) {
1090                                         .csum_good      = csum_good,
1091                                         .dev            = ca->dev_idx,
1092                                         .bucket         = bucket,
1093                                         .bucket_offset  = offset -
1094                                                 bucket_to_sector(ca, ja->buckets[bucket]),
1095                                         .sector         = offset,
1096                                         }, jlist, j);
1097                 mutex_unlock(&jlist->lock);
1098
1099                 switch (ret) {
1100                 case JOURNAL_ENTRY_ADD_OK:
1101                         break;
1102                 case JOURNAL_ENTRY_ADD_OUT_OF_RANGE:
1103                         break;
1104                 default:
1105                         goto err;
1106                 }
1107 next_block:
1108                 pr_debug("next");
1109                 offset          += sectors;
1110                 sectors_read    -= sectors;
1111                 j = ((void *) j) + (sectors << 9);
1112         }
1113
1114 out:
1115         ret = 0;
1116 err:
1117         printbuf_exit(&err);
1118         return ret;
1119 }
1120
1121 static CLOSURE_CALLBACK(bch2_journal_read_device)
1122 {
1123         closure_type(ja, struct journal_device, read);
1124         struct bch_dev *ca = container_of(ja, struct bch_dev, journal);
1125         struct bch_fs *c = ca->fs;
1126         struct journal_list *jlist =
1127                 container_of(cl->parent, struct journal_list, cl);
1128         struct journal_replay *r, **_r;
1129         struct genradix_iter iter;
1130         struct journal_read_buf buf = { NULL, 0 };
1131         unsigned i;
1132         int ret = 0;
1133
1134         if (!ja->nr)
1135                 goto out;
1136
1137         ret = journal_read_buf_realloc(&buf, PAGE_SIZE);
1138         if (ret)
1139                 goto err;
1140
1141         pr_debug("%u journal buckets", ja->nr);
1142
1143         for (i = 0; i < ja->nr; i++) {
1144                 ret = journal_read_bucket(ca, &buf, jlist, i);
1145                 if (ret)
1146                         goto err;
1147         }
1148
1149         ja->sectors_free = ca->mi.bucket_size;
1150
1151         mutex_lock(&jlist->lock);
1152         genradix_for_each_reverse(&c->journal_entries, iter, _r) {
1153                 r = *_r;
1154
1155                 if (!r)
1156                         continue;
1157
1158                 darray_for_each(r->ptrs, i)
1159                         if (i->dev == ca->dev_idx) {
1160                                 unsigned wrote = bucket_remainder(ca, i->sector) +
1161                                         vstruct_sectors(&r->j, c->block_bits);
1162
1163                                 ja->cur_idx = i->bucket;
1164                                 ja->sectors_free = ca->mi.bucket_size - wrote;
1165                                 goto found;
1166                         }
1167         }
1168 found:
1169         mutex_unlock(&jlist->lock);
1170
1171         if (ja->bucket_seq[ja->cur_idx] &&
1172             ja->sectors_free == ca->mi.bucket_size) {
1173 #if 0
1174                 /*
1175                  * Debug code for ZNS support, where we (probably) want to be
1176                  * correlated where we stopped in the journal to the zone write
1177                  * points:
1178                  */
1179                 bch_err(c, "ja->sectors_free == ca->mi.bucket_size");
1180                 bch_err(c, "cur_idx %u/%u", ja->cur_idx, ja->nr);
1181                 for (i = 0; i < 3; i++) {
1182                         unsigned idx = (ja->cur_idx + ja->nr - 1 + i) % ja->nr;
1183
1184                         bch_err(c, "bucket_seq[%u] = %llu", idx, ja->bucket_seq[idx]);
1185                 }
1186 #endif
1187                 ja->sectors_free = 0;
1188         }
1189
1190         /*
1191          * Set dirty_idx to indicate the entire journal is full and needs to be
1192          * reclaimed - journal reclaim will immediately reclaim whatever isn't
1193          * pinned when it first runs:
1194          */
1195         ja->discard_idx = ja->dirty_idx_ondisk =
1196                 ja->dirty_idx = (ja->cur_idx + 1) % ja->nr;
1197 out:
1198         bch_verbose(c, "journal read done on device %s, ret %i", ca->name, ret);
1199         kvfree(buf.data);
1200         percpu_ref_put(&ca->io_ref);
1201         closure_return(cl);
1202         return;
1203 err:
1204         mutex_lock(&jlist->lock);
1205         jlist->ret = ret;
1206         mutex_unlock(&jlist->lock);
1207         goto out;
1208 }
1209
1210 int bch2_journal_read(struct bch_fs *c,
1211                       u64 *last_seq,
1212                       u64 *blacklist_seq,
1213                       u64 *start_seq)
1214 {
1215         struct journal_list jlist;
1216         struct journal_replay *i, **_i, *prev = NULL;
1217         struct genradix_iter radix_iter;
1218         struct printbuf buf = PRINTBUF;
1219         bool degraded = false, last_write_torn = false;
1220         u64 seq;
1221         int ret = 0;
1222
1223         closure_init_stack(&jlist.cl);
1224         mutex_init(&jlist.lock);
1225         jlist.last_seq = 0;
1226         jlist.ret = 0;
1227
1228         for_each_member_device(c, ca) {
1229                 if (!c->opts.fsck &&
1230                     !(bch2_dev_has_data(c, ca) & (1 << BCH_DATA_journal)))
1231                         continue;
1232
1233                 if ((ca->mi.state == BCH_MEMBER_STATE_rw ||
1234                      ca->mi.state == BCH_MEMBER_STATE_ro) &&
1235                     percpu_ref_tryget(&ca->io_ref))
1236                         closure_call(&ca->journal.read,
1237                                      bch2_journal_read_device,
1238                                      system_unbound_wq,
1239                                      &jlist.cl);
1240                 else
1241                         degraded = true;
1242         }
1243
1244         closure_sync(&jlist.cl);
1245
1246         if (jlist.ret)
1247                 return jlist.ret;
1248
1249         *last_seq       = 0;
1250         *start_seq      = 0;
1251         *blacklist_seq  = 0;
1252
1253         /*
1254          * Find most recent flush entry, and ignore newer non flush entries -
1255          * those entries will be blacklisted:
1256          */
1257         genradix_for_each_reverse(&c->journal_entries, radix_iter, _i) {
1258                 enum bkey_invalid_flags flags = BKEY_INVALID_JOURNAL;
1259
1260                 i = *_i;
1261
1262                 if (journal_replay_ignore(i))
1263                         continue;
1264
1265                 if (!*start_seq)
1266                         *blacklist_seq = *start_seq = le64_to_cpu(i->j.seq) + 1;
1267
1268                 if (JSET_NO_FLUSH(&i->j)) {
1269                         i->ignore_blacklisted = true;
1270                         continue;
1271                 }
1272
1273                 if (!last_write_torn && !i->csum_good) {
1274                         last_write_torn = true;
1275                         i->ignore_blacklisted = true;
1276                         continue;
1277                 }
1278
1279                 if (journal_entry_err_on(le64_to_cpu(i->j.last_seq) > le64_to_cpu(i->j.seq),
1280                                          c, le32_to_cpu(i->j.version), &i->j, NULL,
1281                                          jset_last_seq_newer_than_seq,
1282                                          "invalid journal entry: last_seq > seq (%llu > %llu)",
1283                                          le64_to_cpu(i->j.last_seq),
1284                                          le64_to_cpu(i->j.seq)))
1285                         i->j.last_seq = i->j.seq;
1286
1287                 *last_seq       = le64_to_cpu(i->j.last_seq);
1288                 *blacklist_seq  = le64_to_cpu(i->j.seq) + 1;
1289                 break;
1290         }
1291
1292         if (!*start_seq) {
1293                 bch_info(c, "journal read done, but no entries found");
1294                 return 0;
1295         }
1296
1297         if (!*last_seq) {
1298                 fsck_err(c, dirty_but_no_journal_entries_post_drop_nonflushes,
1299                          "journal read done, but no entries found after dropping non-flushes");
1300                 return 0;
1301         }
1302
1303         bch_info(c, "journal read done, replaying entries %llu-%llu",
1304                  *last_seq, *blacklist_seq - 1);
1305
1306         if (*start_seq != *blacklist_seq)
1307                 bch_info(c, "dropped unflushed entries %llu-%llu",
1308                          *blacklist_seq, *start_seq - 1);
1309
1310         /* Drop blacklisted entries and entries older than last_seq: */
1311         genradix_for_each(&c->journal_entries, radix_iter, _i) {
1312                 i = *_i;
1313
1314                 if (journal_replay_ignore(i))
1315                         continue;
1316
1317                 seq = le64_to_cpu(i->j.seq);
1318                 if (seq < *last_seq) {
1319                         journal_replay_free(c, i, false);
1320                         continue;
1321                 }
1322
1323                 if (bch2_journal_seq_is_blacklisted(c, seq, true)) {
1324                         fsck_err_on(!JSET_NO_FLUSH(&i->j), c,
1325                                     jset_seq_blacklisted,
1326                                     "found blacklisted journal entry %llu", seq);
1327                         i->ignore_blacklisted = true;
1328                 }
1329         }
1330
1331         /* Check for missing entries: */
1332         seq = *last_seq;
1333         genradix_for_each(&c->journal_entries, radix_iter, _i) {
1334                 i = *_i;
1335
1336                 if (journal_replay_ignore(i))
1337                         continue;
1338
1339                 BUG_ON(seq > le64_to_cpu(i->j.seq));
1340
1341                 while (seq < le64_to_cpu(i->j.seq)) {
1342                         u64 missing_start, missing_end;
1343                         struct printbuf buf1 = PRINTBUF, buf2 = PRINTBUF;
1344
1345                         while (seq < le64_to_cpu(i->j.seq) &&
1346                                bch2_journal_seq_is_blacklisted(c, seq, false))
1347                                 seq++;
1348
1349                         if (seq == le64_to_cpu(i->j.seq))
1350                                 break;
1351
1352                         missing_start = seq;
1353
1354                         while (seq < le64_to_cpu(i->j.seq) &&
1355                                !bch2_journal_seq_is_blacklisted(c, seq, false))
1356                                 seq++;
1357
1358                         if (prev) {
1359                                 bch2_journal_ptrs_to_text(&buf1, c, prev);
1360                                 prt_printf(&buf1, " size %zu", vstruct_sectors(&prev->j, c->block_bits));
1361                         } else
1362                                 prt_printf(&buf1, "(none)");
1363                         bch2_journal_ptrs_to_text(&buf2, c, i);
1364
1365                         missing_end = seq - 1;
1366                         fsck_err(c, journal_entries_missing,
1367                                  "journal entries %llu-%llu missing! (replaying %llu-%llu)\n"
1368                                  "  prev at %s\n"
1369                                  "  next at %s",
1370                                  missing_start, missing_end,
1371                                  *last_seq, *blacklist_seq - 1,
1372                                  buf1.buf, buf2.buf);
1373
1374                         printbuf_exit(&buf1);
1375                         printbuf_exit(&buf2);
1376                 }
1377
1378                 prev = i;
1379                 seq++;
1380         }
1381
1382         genradix_for_each(&c->journal_entries, radix_iter, _i) {
1383                 struct bch_replicas_padded replicas = {
1384                         .e.data_type = BCH_DATA_journal,
1385                         .e.nr_required = 1,
1386                 };
1387
1388                 i = *_i;
1389                 if (journal_replay_ignore(i))
1390                         continue;
1391
1392                 darray_for_each(i->ptrs, ptr) {
1393                         struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
1394
1395                         if (!ptr->csum_good)
1396                                 bch_err_dev_offset(ca, ptr->sector,
1397                                                    "invalid journal checksum, seq %llu%s",
1398                                                    le64_to_cpu(i->j.seq),
1399                                                    i->csum_good ? " (had good copy on another device)" : "");
1400                 }
1401
1402                 ret = jset_validate(c,
1403                                     bch_dev_bkey_exists(c, i->ptrs.data[0].dev),
1404                                     &i->j,
1405                                     i->ptrs.data[0].sector,
1406                                     READ);
1407                 if (ret)
1408                         goto err;
1409
1410                 darray_for_each(i->ptrs, ptr)
1411                         replicas.e.devs[replicas.e.nr_devs++] = ptr->dev;
1412
1413                 bch2_replicas_entry_sort(&replicas.e);
1414
1415                 printbuf_reset(&buf);
1416                 bch2_replicas_entry_to_text(&buf, &replicas.e);
1417
1418                 if (!degraded &&
1419                     !bch2_replicas_marked(c, &replicas.e) &&
1420                     (le64_to_cpu(i->j.seq) == *last_seq ||
1421                      fsck_err(c, journal_entry_replicas_not_marked,
1422                               "superblock not marked as containing replicas for journal entry %llu\n  %s",
1423                               le64_to_cpu(i->j.seq), buf.buf))) {
1424                         ret = bch2_mark_replicas(c, &replicas.e);
1425                         if (ret)
1426                                 goto err;
1427                 }
1428         }
1429 err:
1430 fsck_err:
1431         printbuf_exit(&buf);
1432         return ret;
1433 }
1434
1435 /* journal write: */
1436
1437 static void __journal_write_alloc(struct journal *j,
1438                                   struct journal_buf *w,
1439                                   struct dev_alloc_list *devs_sorted,
1440                                   unsigned sectors,
1441                                   unsigned *replicas,
1442                                   unsigned replicas_want)
1443 {
1444         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1445         struct journal_device *ja;
1446         struct bch_dev *ca;
1447         unsigned i;
1448
1449         if (*replicas >= replicas_want)
1450                 return;
1451
1452         for (i = 0; i < devs_sorted->nr; i++) {
1453                 ca = rcu_dereference(c->devs[devs_sorted->devs[i]]);
1454                 if (!ca)
1455                         continue;
1456
1457                 ja = &ca->journal;
1458
1459                 /*
1460                  * Check that we can use this device, and aren't already using
1461                  * it:
1462                  */
1463                 if (!ca->mi.durability ||
1464                     ca->mi.state != BCH_MEMBER_STATE_rw ||
1465                     !ja->nr ||
1466                     bch2_bkey_has_device_c(bkey_i_to_s_c(&w->key), ca->dev_idx) ||
1467                     sectors > ja->sectors_free)
1468                         continue;
1469
1470                 bch2_dev_stripe_increment(ca, &j->wp.stripe);
1471
1472                 bch2_bkey_append_ptr(&w->key,
1473                         (struct bch_extent_ptr) {
1474                                   .offset = bucket_to_sector(ca,
1475                                         ja->buckets[ja->cur_idx]) +
1476                                         ca->mi.bucket_size -
1477                                         ja->sectors_free,
1478                                   .dev = ca->dev_idx,
1479                 });
1480
1481                 ja->sectors_free -= sectors;
1482                 ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq);
1483
1484                 *replicas += ca->mi.durability;
1485
1486                 if (*replicas >= replicas_want)
1487                         break;
1488         }
1489 }
1490
1491 /**
1492  * journal_write_alloc - decide where to write next journal entry
1493  *
1494  * @j:          journal object
1495  * @w:          journal buf (entry to be written)
1496  *
1497  * Returns: 0 on success, or -EROFS on failure
1498  */
1499 static int journal_write_alloc(struct journal *j, struct journal_buf *w)
1500 {
1501         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1502         struct bch_devs_mask devs;
1503         struct journal_device *ja;
1504         struct bch_dev *ca;
1505         struct dev_alloc_list devs_sorted;
1506         unsigned sectors = vstruct_sectors(w->data, c->block_bits);
1507         unsigned target = c->opts.metadata_target ?:
1508                 c->opts.foreground_target;
1509         unsigned i, replicas = 0, replicas_want =
1510                 READ_ONCE(c->opts.metadata_replicas);
1511         unsigned replicas_need = min_t(unsigned, replicas_want,
1512                                        READ_ONCE(c->opts.metadata_replicas_required));
1513
1514         rcu_read_lock();
1515 retry:
1516         devs = target_rw_devs(c, BCH_DATA_journal, target);
1517
1518         devs_sorted = bch2_dev_alloc_list(c, &j->wp.stripe, &devs);
1519
1520         __journal_write_alloc(j, w, &devs_sorted,
1521                               sectors, &replicas, replicas_want);
1522
1523         if (replicas >= replicas_want)
1524                 goto done;
1525
1526         for (i = 0; i < devs_sorted.nr; i++) {
1527                 ca = rcu_dereference(c->devs[devs_sorted.devs[i]]);
1528                 if (!ca)
1529                         continue;
1530
1531                 ja = &ca->journal;
1532
1533                 if (sectors > ja->sectors_free &&
1534                     sectors <= ca->mi.bucket_size &&
1535                     bch2_journal_dev_buckets_available(j, ja,
1536                                         journal_space_discarded)) {
1537                         ja->cur_idx = (ja->cur_idx + 1) % ja->nr;
1538                         ja->sectors_free = ca->mi.bucket_size;
1539
1540                         /*
1541                          * ja->bucket_seq[ja->cur_idx] must always have
1542                          * something sensible:
1543                          */
1544                         ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq);
1545                 }
1546         }
1547
1548         __journal_write_alloc(j, w, &devs_sorted,
1549                               sectors, &replicas, replicas_want);
1550
1551         if (replicas < replicas_want && target) {
1552                 /* Retry from all devices: */
1553                 target = 0;
1554                 goto retry;
1555         }
1556 done:
1557         rcu_read_unlock();
1558
1559         BUG_ON(bkey_val_u64s(&w->key.k) > BCH_REPLICAS_MAX);
1560
1561         return replicas >= replicas_need ? 0 : -EROFS;
1562 }
1563
1564 static void journal_buf_realloc(struct journal *j, struct journal_buf *buf)
1565 {
1566         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1567
1568         /* we aren't holding j->lock: */
1569         unsigned new_size = READ_ONCE(j->buf_size_want);
1570         void *new_buf;
1571
1572         if (buf->buf_size >= new_size)
1573                 return;
1574
1575         size_t btree_write_buffer_size = new_size / 64;
1576
1577         if (bch2_btree_write_buffer_resize(c, btree_write_buffer_size))
1578                 return;
1579
1580         new_buf = kvmalloc(new_size, GFP_NOFS|__GFP_NOWARN);
1581         if (!new_buf)
1582                 return;
1583
1584         memcpy(new_buf, buf->data, buf->buf_size);
1585
1586         spin_lock(&j->lock);
1587         swap(buf->data,         new_buf);
1588         swap(buf->buf_size,     new_size);
1589         spin_unlock(&j->lock);
1590
1591         kvfree(new_buf);
1592 }
1593
1594 static inline struct journal_buf *journal_last_unwritten_buf(struct journal *j)
1595 {
1596         return j->buf + (journal_last_unwritten_seq(j) & JOURNAL_BUF_MASK);
1597 }
1598
1599 static CLOSURE_CALLBACK(journal_write_done)
1600 {
1601         closure_type(w, struct journal_buf, io);
1602         struct journal *j = container_of(w, struct journal, buf[w->idx]);
1603         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1604         struct bch_replicas_padded replicas;
1605         union journal_res_state old, new;
1606         u64 v, seq = le64_to_cpu(w->data->seq);
1607         int err = 0;
1608
1609         bch2_time_stats_update(!JSET_NO_FLUSH(w->data)
1610                                ? j->flush_write_time
1611                                : j->noflush_write_time, j->write_start_time);
1612
1613         if (!w->devs_written.nr) {
1614                 bch_err(c, "unable to write journal to sufficient devices");
1615                 err = -EIO;
1616         } else {
1617                 bch2_devlist_to_replicas(&replicas.e, BCH_DATA_journal,
1618                                          w->devs_written);
1619                 if (bch2_mark_replicas(c, &replicas.e))
1620                         err = -EIO;
1621         }
1622
1623         if (err)
1624                 bch2_fatal_error(c);
1625
1626         closure_debug_destroy(cl);
1627
1628         spin_lock(&j->lock);
1629         if (seq >= j->pin.front)
1630                 journal_seq_pin(j, seq)->devs = w->devs_written;
1631         if (err && (!j->err_seq || seq < j->err_seq))
1632                 j->err_seq      = seq;
1633         w->write_done = true;
1634
1635         bool completed = false;
1636
1637         for (seq = journal_last_unwritten_seq(j);
1638              seq <= journal_cur_seq(j);
1639              seq++) {
1640                 w = j->buf + (seq & JOURNAL_BUF_MASK);
1641                 if (!w->write_done)
1642                         break;
1643
1644                 if (!j->err_seq && !JSET_NO_FLUSH(w->data)) {
1645                         j->flushed_seq_ondisk = seq;
1646                         j->last_seq_ondisk = w->last_seq;
1647
1648                         bch2_do_discards(c);
1649                         closure_wake_up(&c->freelist_wait);
1650                         bch2_reset_alloc_cursors(c);
1651                 }
1652
1653                 j->seq_ondisk = seq;
1654
1655                 /*
1656                  * Updating last_seq_ondisk may let bch2_journal_reclaim_work() discard
1657                  * more buckets:
1658                  *
1659                  * Must come before signaling write completion, for
1660                  * bch2_fs_journal_stop():
1661                  */
1662                 if (j->watermark != BCH_WATERMARK_stripe)
1663                         journal_reclaim_kick(&c->journal);
1664
1665                 v = atomic64_read(&j->reservations.counter);
1666                 do {
1667                         old.v = new.v = v;
1668                         BUG_ON(journal_state_count(new, new.unwritten_idx));
1669                         BUG_ON(new.unwritten_idx != (seq & JOURNAL_BUF_MASK));
1670
1671                         new.unwritten_idx++;
1672                 } while ((v = atomic64_cmpxchg(&j->reservations.counter, old.v, new.v)) != old.v);
1673
1674                 closure_wake_up(&w->wait);
1675                 completed = true;
1676         }
1677
1678         if (completed) {
1679                 bch2_journal_reclaim_fast(j);
1680                 bch2_journal_space_available(j);
1681
1682                 track_event_change(&c->times[BCH_TIME_blocked_journal_max_in_flight], false);
1683
1684                 journal_wake(j);
1685         }
1686
1687         if (journal_last_unwritten_seq(j) == journal_cur_seq(j) &&
1688                    new.cur_entry_offset < JOURNAL_ENTRY_CLOSED_VAL) {
1689                 struct journal_buf *buf = journal_cur_buf(j);
1690                 long delta = buf->expires - jiffies;
1691
1692                 /*
1693                  * We don't close a journal entry to write it while there's
1694                  * previous entries still in flight - the current journal entry
1695                  * might want to be written now:
1696                  */
1697                 mod_delayed_work(j->wq, &j->write_work, max(0L, delta));
1698         }
1699
1700         spin_unlock(&j->lock);
1701 }
1702
1703 static void journal_write_endio(struct bio *bio)
1704 {
1705         struct journal_bio *jbio = container_of(bio, struct journal_bio, bio);
1706         struct bch_dev *ca = jbio->ca;
1707         struct journal *j = &ca->fs->journal;
1708         struct journal_buf *w = j->buf + jbio->buf_idx;
1709
1710         if (bch2_dev_io_err_on(bio->bi_status, ca, BCH_MEMBER_ERROR_write,
1711                                "error writing journal entry %llu: %s",
1712                                le64_to_cpu(w->data->seq),
1713                                bch2_blk_status_to_str(bio->bi_status)) ||
1714             bch2_meta_write_fault("journal")) {
1715                 unsigned long flags;
1716
1717                 spin_lock_irqsave(&j->err_lock, flags);
1718                 bch2_dev_list_drop_dev(&w->devs_written, ca->dev_idx);
1719                 spin_unlock_irqrestore(&j->err_lock, flags);
1720         }
1721
1722         closure_put(&w->io);
1723         percpu_ref_put(&ca->io_ref);
1724 }
1725
1726 static CLOSURE_CALLBACK(journal_write_submit)
1727 {
1728         closure_type(w, struct journal_buf, io);
1729         struct journal *j = container_of(w, struct journal, buf[w->idx]);
1730         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1731         unsigned sectors = vstruct_sectors(w->data, c->block_bits);
1732
1733         extent_for_each_ptr(bkey_i_to_s_extent(&w->key), ptr) {
1734                 struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
1735                 struct journal_device *ja = &ca->journal;
1736
1737                 if (!percpu_ref_tryget(&ca->io_ref)) {
1738                         /* XXX: fix this */
1739                         bch_err(c, "missing device for journal write\n");
1740                         continue;
1741                 }
1742
1743                 this_cpu_add(ca->io_done->sectors[WRITE][BCH_DATA_journal],
1744                              sectors);
1745
1746                 struct bio *bio = &ja->bio[w->idx]->bio;
1747                 bio_reset(bio, ca->disk_sb.bdev, REQ_OP_WRITE|REQ_SYNC|REQ_META);
1748                 bio->bi_iter.bi_sector  = ptr->offset;
1749                 bio->bi_end_io          = journal_write_endio;
1750                 bio->bi_private         = ca;
1751
1752                 BUG_ON(bio->bi_iter.bi_sector == ca->prev_journal_sector);
1753                 ca->prev_journal_sector = bio->bi_iter.bi_sector;
1754
1755                 if (!JSET_NO_FLUSH(w->data))
1756                         bio->bi_opf    |= REQ_FUA;
1757                 if (!JSET_NO_FLUSH(w->data) && !w->separate_flush)
1758                         bio->bi_opf    |= REQ_PREFLUSH;
1759
1760                 bch2_bio_map(bio, w->data, sectors << 9);
1761
1762                 trace_and_count(c, journal_write, bio);
1763                 closure_bio_submit(bio, cl);
1764
1765                 ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq);
1766         }
1767
1768         continue_at(cl, journal_write_done, j->wq);
1769 }
1770
1771 static CLOSURE_CALLBACK(journal_write_preflush)
1772 {
1773         closure_type(w, struct journal_buf, io);
1774         struct journal *j = container_of(w, struct journal, buf[w->idx]);
1775         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1776
1777         if (j->seq_ondisk + 1 != le64_to_cpu(w->data->seq)) {
1778                 spin_lock(&j->lock);
1779                 closure_wait(&j->async_wait, cl);
1780                 spin_unlock(&j->lock);
1781
1782                 continue_at(cl, journal_write_preflush, j->wq);
1783                 return;
1784         }
1785
1786         if (w->separate_flush) {
1787                 for_each_rw_member(c, ca) {
1788                         percpu_ref_get(&ca->io_ref);
1789
1790                         struct journal_device *ja = &ca->journal;
1791                         struct bio *bio = &ja->bio[w->idx]->bio;
1792                         bio_reset(bio, ca->disk_sb.bdev,
1793                                   REQ_OP_WRITE|REQ_SYNC|REQ_META|REQ_PREFLUSH);
1794                         bio->bi_end_io          = journal_write_endio;
1795                         bio->bi_private         = ca;
1796                         closure_bio_submit(bio, cl);
1797                 }
1798
1799                 continue_at(cl, journal_write_submit, j->wq);
1800         } else {
1801                 /*
1802                  * no need to punt to another work item if we're not waiting on
1803                  * preflushes
1804                  */
1805                 journal_write_submit(&cl->work);
1806         }
1807 }
1808
1809 static int bch2_journal_write_prep(struct journal *j, struct journal_buf *w)
1810 {
1811         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1812         struct jset_entry *start, *end;
1813         struct jset *jset = w->data;
1814         struct journal_keys_to_wb wb = { NULL };
1815         unsigned sectors, bytes, u64s;
1816         unsigned long btree_roots_have = 0;
1817         bool validate_before_checksum = false;
1818         u64 seq = le64_to_cpu(jset->seq);
1819         int ret;
1820
1821         /*
1822          * Simple compaction, dropping empty jset_entries (from journal
1823          * reservations that weren't fully used) and merging jset_entries that
1824          * can be.
1825          *
1826          * If we wanted to be really fancy here, we could sort all the keys in
1827          * the jset and drop keys that were overwritten - probably not worth it:
1828          */
1829         vstruct_for_each(jset, i) {
1830                 unsigned u64s = le16_to_cpu(i->u64s);
1831
1832                 /* Empty entry: */
1833                 if (!u64s)
1834                         continue;
1835
1836                 /*
1837                  * New btree roots are set by journalling them; when the journal
1838                  * entry gets written we have to propagate them to
1839                  * c->btree_roots
1840                  *
1841                  * But, every journal entry we write has to contain all the
1842                  * btree roots (at least for now); so after we copy btree roots
1843                  * to c->btree_roots we have to get any missing btree roots and
1844                  * add them to this journal entry:
1845                  */
1846                 switch (i->type) {
1847                 case BCH_JSET_ENTRY_btree_root:
1848                         bch2_journal_entry_to_btree_root(c, i);
1849                         __set_bit(i->btree_id, &btree_roots_have);
1850                         break;
1851                 case BCH_JSET_ENTRY_write_buffer_keys:
1852                         EBUG_ON(!w->need_flush_to_write_buffer);
1853
1854                         if (!wb.wb)
1855                                 bch2_journal_keys_to_write_buffer_start(c, &wb, seq);
1856
1857                         jset_entry_for_each_key(i, k) {
1858                                 ret = bch2_journal_key_to_wb(c, &wb, i->btree_id, k);
1859                                 if (ret) {
1860                                         bch2_fs_fatal_error(c, "flushing journal keys to btree write buffer: %s",
1861                                                             bch2_err_str(ret));
1862                                         bch2_journal_keys_to_write_buffer_end(c, &wb);
1863                                         return ret;
1864                                 }
1865                         }
1866                         i->type = BCH_JSET_ENTRY_btree_keys;
1867                         break;
1868                 }
1869         }
1870
1871         if (wb.wb)
1872                 bch2_journal_keys_to_write_buffer_end(c, &wb);
1873
1874         spin_lock(&c->journal.lock);
1875         w->need_flush_to_write_buffer = false;
1876         spin_unlock(&c->journal.lock);
1877
1878         start = end = vstruct_last(jset);
1879
1880         end     = bch2_btree_roots_to_journal_entries(c, end, btree_roots_have);
1881
1882         struct jset_entry_datetime *d =
1883                 container_of(jset_entry_init(&end, sizeof(*d)), struct jset_entry_datetime, entry);
1884         d->entry.type   = BCH_JSET_ENTRY_datetime;
1885         d->seconds      = cpu_to_le64(ktime_get_real_seconds());
1886
1887         bch2_journal_super_entries_add_common(c, &end, seq);
1888         u64s    = (u64 *) end - (u64 *) start;
1889
1890         WARN_ON(u64s > j->entry_u64s_reserved);
1891
1892         le32_add_cpu(&jset->u64s, u64s);
1893
1894         sectors = vstruct_sectors(jset, c->block_bits);
1895         bytes   = vstruct_bytes(jset);
1896
1897         if (sectors > w->sectors) {
1898                 bch2_fs_fatal_error(c, ": journal write overran available space, %zu > %u (extra %u reserved %u/%u)",
1899                                     vstruct_bytes(jset), w->sectors << 9,
1900                                     u64s, w->u64s_reserved, j->entry_u64s_reserved);
1901                 return -EINVAL;
1902         }
1903
1904         jset->magic             = cpu_to_le64(jset_magic(c));
1905         jset->version           = cpu_to_le32(c->sb.version);
1906
1907         SET_JSET_BIG_ENDIAN(jset, CPU_BIG_ENDIAN);
1908         SET_JSET_CSUM_TYPE(jset, bch2_meta_checksum_type(c));
1909
1910         if (!JSET_NO_FLUSH(jset) && journal_entry_empty(jset))
1911                 j->last_empty_seq = seq;
1912
1913         if (bch2_csum_type_is_encryption(JSET_CSUM_TYPE(jset)))
1914                 validate_before_checksum = true;
1915
1916         if (le32_to_cpu(jset->version) < bcachefs_metadata_version_current)
1917                 validate_before_checksum = true;
1918
1919         if (validate_before_checksum &&
1920             (ret = jset_validate(c, NULL, jset, 0, WRITE)))
1921                 return ret;
1922
1923         ret = bch2_encrypt(c, JSET_CSUM_TYPE(jset), journal_nonce(jset),
1924                     jset->encrypted_start,
1925                     vstruct_end(jset) - (void *) jset->encrypted_start);
1926         if (bch2_fs_fatal_err_on(ret, c, "decrypting journal entry: %s", bch2_err_str(ret)))
1927                 return ret;
1928
1929         jset->csum = csum_vstruct(c, JSET_CSUM_TYPE(jset),
1930                                   journal_nonce(jset), jset);
1931
1932         if (!validate_before_checksum &&
1933             (ret = jset_validate(c, NULL, jset, 0, WRITE)))
1934                 return ret;
1935
1936         memset((void *) jset + bytes, 0, (sectors << 9) - bytes);
1937         return 0;
1938 }
1939
1940 static int bch2_journal_write_pick_flush(struct journal *j, struct journal_buf *w)
1941 {
1942         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1943         int error = bch2_journal_error(j);
1944
1945         /*
1946          * If the journal is in an error state - we did an emergency shutdown -
1947          * we prefer to continue doing journal writes. We just mark them as
1948          * noflush so they'll never be used, but they'll still be visible by the
1949          * list_journal tool - this helps in debugging.
1950          *
1951          * There's a caveat: the first journal write after marking the
1952          * superblock dirty must always be a flush write, because on startup
1953          * from a clean shutdown we didn't necessarily read the journal and the
1954          * new journal write might overwrite whatever was in the journal
1955          * previously - we can't leave the journal without any flush writes in
1956          * it.
1957          *
1958          * So if we're in an error state, and we're still starting up, we don't
1959          * write anything at all.
1960          */
1961         if (error && test_bit(JOURNAL_NEED_FLUSH_WRITE, &j->flags))
1962                 return -EIO;
1963
1964         if (error ||
1965             w->noflush ||
1966             (!w->must_flush &&
1967              (jiffies - j->last_flush_write) < msecs_to_jiffies(c->opts.journal_flush_delay) &&
1968              test_bit(JOURNAL_MAY_SKIP_FLUSH, &j->flags))) {
1969                 w->noflush = true;
1970                 SET_JSET_NO_FLUSH(w->data, true);
1971                 w->data->last_seq       = 0;
1972                 w->last_seq             = 0;
1973
1974                 j->nr_noflush_writes++;
1975         } else {
1976                 w->must_flush = true;
1977                 j->last_flush_write = jiffies;
1978                 j->nr_flush_writes++;
1979                 clear_bit(JOURNAL_NEED_FLUSH_WRITE, &j->flags);
1980         }
1981
1982         return 0;
1983 }
1984
1985 CLOSURE_CALLBACK(bch2_journal_write)
1986 {
1987         closure_type(w, struct journal_buf, io);
1988         struct journal *j = container_of(w, struct journal, buf[w->idx]);
1989         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1990         struct bch_replicas_padded replicas;
1991         struct printbuf journal_debug_buf = PRINTBUF;
1992         unsigned nr_rw_members = 0;
1993         int ret;
1994
1995         for_each_rw_member(c, ca)
1996                 nr_rw_members++;
1997
1998         BUG_ON(BCH_SB_CLEAN(c->disk_sb.sb));
1999         BUG_ON(!w->write_started);
2000         BUG_ON(w->write_allocated);
2001         BUG_ON(w->write_done);
2002
2003         j->write_start_time = local_clock();
2004
2005         spin_lock(&j->lock);
2006         if (nr_rw_members > 1)
2007                 w->separate_flush = true;
2008
2009         ret = bch2_journal_write_pick_flush(j, w);
2010         spin_unlock(&j->lock);
2011         if (ret)
2012                 goto err;
2013
2014         mutex_lock(&j->buf_lock);
2015         journal_buf_realloc(j, w);
2016
2017         ret = bch2_journal_write_prep(j, w);
2018         mutex_unlock(&j->buf_lock);
2019         if (ret)
2020                 goto err;
2021
2022         j->entry_bytes_written += vstruct_bytes(w->data);
2023
2024         while (1) {
2025                 spin_lock(&j->lock);
2026                 ret = journal_write_alloc(j, w);
2027                 if (!ret || !j->can_discard)
2028                         break;
2029
2030                 spin_unlock(&j->lock);
2031                 bch2_journal_do_discards(j);
2032         }
2033
2034         if (ret) {
2035                 __bch2_journal_debug_to_text(&journal_debug_buf, j);
2036                 spin_unlock(&j->lock);
2037                 bch_err(c, "Unable to allocate journal write:\n%s",
2038                         journal_debug_buf.buf);
2039                 printbuf_exit(&journal_debug_buf);
2040                 goto err;
2041         }
2042
2043         /*
2044          * write is allocated, no longer need to account for it in
2045          * bch2_journal_space_available():
2046          */
2047         w->sectors = 0;
2048         w->write_allocated = true;
2049
2050         /*
2051          * journal entry has been compacted and allocated, recalculate space
2052          * available:
2053          */
2054         bch2_journal_space_available(j);
2055         bch2_journal_do_writes(j);
2056         spin_unlock(&j->lock);
2057
2058         w->devs_written = bch2_bkey_devs(bkey_i_to_s_c(&w->key));
2059
2060         if (c->opts.nochanges)
2061                 goto no_io;
2062
2063         /*
2064          * Mark journal replicas before we submit the write to guarantee
2065          * recovery will find the journal entries after a crash.
2066          */
2067         bch2_devlist_to_replicas(&replicas.e, BCH_DATA_journal,
2068                                  w->devs_written);
2069         ret = bch2_mark_replicas(c, &replicas.e);
2070         if (ret)
2071                 goto err;
2072
2073         if (!JSET_NO_FLUSH(w->data))
2074                 continue_at(cl, journal_write_preflush, j->wq);
2075         else
2076                 continue_at(cl, journal_write_submit, j->wq);
2077         return;
2078 no_io:
2079         continue_at(cl, journal_write_done, j->wq);
2080         return;
2081 err:
2082         bch2_fatal_error(c);
2083         continue_at(cl, journal_write_done, j->wq);
2084 }