fs/bcachefs/fs-io-buffered.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #ifndef NO_BCACHEFS_FS
   3
   4 #include "bcachefs.h"
   5 #include "alloc_foreground.h"
   6 #include "bkey_buf.h"
   7 #include "fs-io.h"
   8 #include "fs-io-buffered.h"
   9 #include "fs-io-direct.h"
  10 #include "fs-io-pagecache.h"
  11 #include "io_read.h"
  12 #include "io_write.h"
  13
  14 #include <linux/backing-dev.h>
  15 #include <linux/pagemap.h>
  16 #include <linux/writeback.h>
  17
  18 static inline bool bio_full(struct bio *bio, unsigned len)
  19 {
  20         if (bio->bi_vcnt >= bio->bi_max_vecs)
  21                 return true;
  22         if (bio->bi_iter.bi_size > UINT_MAX - len)
  23                 return true;
  24         return false;
  25 }
  26
  27 /* readpage(s): */
  28
  29 static void bch2_readpages_end_io(struct bio *bio)
  30 {
  31         struct folio_iter fi;
  32
  33         bio_for_each_folio_all(fi, bio) {
  34                 if (!bio->bi_status) {
  35                         folio_mark_uptodate(fi.folio);
  36                 } else {
  37                         folio_clear_uptodate(fi.folio);
  38                         folio_set_error(fi.folio);
  39                 }
  40                 folio_unlock(fi.folio);
  41         }
  42
  43         bio_put(bio);
  44 }
  45
  46 struct readpages_iter {
  47         struct address_space    *mapping;
  48         unsigned                idx;
  49         folios                  folios;
  50 };
  51
  52 static int readpages_iter_init(struct readpages_iter *iter,
  53                                struct readahead_control *ractl)
  54 {
  55         struct folio *folio;
  56
  57         *iter = (struct readpages_iter) { ractl->mapping };
  58
  59         while ((folio = __readahead_folio(ractl))) {
  60                 if (!bch2_folio_create(folio, GFP_KERNEL) ||
  61                     darray_push(&iter->folios, folio)) {
  62                         bch2_folio_release(folio);
  63                         ractl->_nr_pages += folio_nr_pages(folio);
  64                         ractl->_index -= folio_nr_pages(folio);
  65                         return iter->folios.nr ? 0 : -ENOMEM;
  66                 }
  67
  68                 folio_put(folio);
  69         }
  70
  71         return 0;
  72 }
  73
  74 static inline struct folio *readpage_iter_peek(struct readpages_iter *iter)
  75 {
  76         if (iter->idx >= iter->folios.nr)
  77                 return NULL;
  78         return iter->folios.data[iter->idx];
  79 }
  80
  81 static inline void readpage_iter_advance(struct readpages_iter *iter)
  82 {
  83         iter->idx++;
  84 }
  85
  86 static bool extent_partial_reads_expensive(struct bkey_s_c k)
  87 {
  88         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
  89         struct bch_extent_crc_unpacked crc;
  90         const union bch_extent_entry *i;
  91
  92         bkey_for_each_crc(k.k, ptrs, crc, i)
  93                 if (crc.csum_type || crc.compression_type)
  94                         return true;
  95         return false;
  96 }
  97
  98 static int readpage_bio_extend(struct btree_trans *trans,
  99                                struct readpages_iter *iter,
 100                                struct bio *bio,
 101                                unsigned sectors_this_extent,
 102                                bool get_more)
 103 {
 104         /* Don't hold btree locks while allocating memory: */
 105         bch2_trans_unlock(trans);
 106
 107         while (bio_sectors(bio) < sectors_this_extent &&
 108                bio->bi_vcnt < bio->bi_max_vecs) {
 109                 struct folio *folio = readpage_iter_peek(iter);
 110                 int ret;
 111
 112                 if (folio) {
 113                         readpage_iter_advance(iter);
 114                 } else {
 115                         pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;
 116
 117                         if (!get_more)
 118                                 break;
 119
 120                         folio = xa_load(&iter->mapping->i_pages, folio_offset);
 121                         if (folio && !xa_is_value(folio))
 122                                 break;
 123
 124                         folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0);
 125                         if (!folio)
 126                                 break;
 127
 128                         if (!__bch2_folio_create(folio, GFP_KERNEL)) {
 129                                 folio_put(folio);
 130                                 break;
 131                         }
 132
 133                         ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL);
 134                         if (ret) {
 135                                 __bch2_folio_release(folio);
 136                                 folio_put(folio);
 137                                 break;
 138                         }
 139
 140                         folio_put(folio);
 141                 }
 142
 143                 BUG_ON(folio_sector(folio) != bio_end_sector(bio));
 144
 145                 BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0));
 146         }
 147
 148         return bch2_trans_relock(trans);
 149 }
 150
 151 static void bchfs_read(struct btree_trans *trans,
 152                        struct bch_read_bio *rbio,
 153                        subvol_inum inum,
 154                        struct readpages_iter *readpages_iter)
 155 {
 156         struct bch_fs *c = trans->c;
 157         struct btree_iter iter;
 158         struct bkey_buf sk;
 159         int flags = BCH_READ_RETRY_IF_STALE|
 160                 BCH_READ_MAY_PROMOTE;
 161         u32 snapshot;
 162         int ret = 0;
 163
 164         rbio->c = c;
 165         rbio->start_time = local_clock();
 166         rbio->subvol = inum.subvol;
 167
 168         bch2_bkey_buf_init(&sk);
 169 retry:
 170         bch2_trans_begin(trans);
 171         iter = (struct btree_iter) { NULL };
 172
 173         ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
 174         if (ret)
 175                 goto err;
 176
 177         bch2_trans_iter_init(trans, &iter, BTREE_ID_extents,
 178                              SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot),
 179                              BTREE_ITER_SLOTS);
 180         while (1) {
 181                 struct bkey_s_c k;
 182                 unsigned bytes, sectors, offset_into_extent;
 183                 enum btree_id data_btree = BTREE_ID_extents;
 184
 185                 /*
 186                  * read_extent -> io_time_reset may cause a transaction restart
 187                  * without returning an error, we need to check for that here:
 188                  */
 189                 ret = bch2_trans_relock(trans);
 190                 if (ret)
 191                         break;
 192
 193                 bch2_btree_iter_set_pos(&iter,
 194                                 POS(inum.inum, rbio->bio.bi_iter.bi_sector));
 195
 196                 k = bch2_btree_iter_peek_slot(&iter);
 197                 ret = bkey_err(k);
 198                 if (ret)
 199                         break;
 200
 201                 offset_into_extent = iter.pos.offset -
 202                         bkey_start_offset(k.k);
 203                 sectors = k.k->size - offset_into_extent;
 204
 205                 bch2_bkey_buf_reassemble(&sk, c, k);
 206
 207                 ret = bch2_read_indirect_extent(trans, &data_btree,
 208                                         &offset_into_extent, &sk);
 209                 if (ret)
 210                         break;
 211
 212                 k = bkey_i_to_s_c(sk.k);
 213
 214                 sectors = min(sectors, k.k->size - offset_into_extent);
 215
 216                 if (readpages_iter) {
 217                         ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors,
 218                                                   extent_partial_reads_expensive(k));
 219                         if (ret)
 220                                 break;
 221                 }
 222
 223                 bytes = min(sectors, bio_sectors(&rbio->bio)) << 9;
 224                 swap(rbio->bio.bi_iter.bi_size, bytes);
 225
 226                 if (rbio->bio.bi_iter.bi_size == bytes)
 227                         flags |= BCH_READ_LAST_FRAGMENT;
 228
 229                 bch2_bio_page_state_set(&rbio->bio, k);
 230
 231                 bch2_read_extent(trans, rbio, iter.pos,
 232                                  data_btree, k, offset_into_extent, flags);
 233
 234                 if (flags & BCH_READ_LAST_FRAGMENT)
 235                         break;
 236
 237                 swap(rbio->bio.bi_iter.bi_size, bytes);
 238                 bio_advance(&rbio->bio, bytes);
 239
 240                 ret = btree_trans_too_many_iters(trans);
 241                 if (ret)
 242                         break;
 243         }
 244 err:
 245         bch2_trans_iter_exit(trans, &iter);
 246
 247         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 248                 goto retry;
 249
 250         if (ret) {
 251                 bch_err_inum_offset_ratelimited(c,
 252                                 iter.pos.inode,
 253                                 iter.pos.offset << 9,
 254                                 "read error %i from btree lookup", ret);
 255                 rbio->bio.bi_status = BLK_STS_IOERR;
 256                 bio_endio(&rbio->bio);
 257         }
 258
 259         bch2_bkey_buf_exit(&sk, c);
 260 }
 261
 262 void bch2_readahead(struct readahead_control *ractl)
 263 {
 264         struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host);
 265         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 266         struct bch_io_opts opts;
 267         struct btree_trans *trans = bch2_trans_get(c);
 268         struct folio *folio;
 269         struct readpages_iter readpages_iter;
 270
 271         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
 272
 273         int ret = readpages_iter_init(&readpages_iter, ractl);
 274         if (ret)
 275                 return;
 276
 277         bch2_pagecache_add_get(inode);
 278
 279         while ((folio = readpage_iter_peek(&readpages_iter))) {
 280                 unsigned n = min_t(unsigned,
 281                                    readpages_iter.folios.nr -
 282                                    readpages_iter.idx,
 283                                    BIO_MAX_VECS);
 284                 struct bch_read_bio *rbio =
 285                         rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ,
 286                                                    GFP_KERNEL, &c->bio_read),
 287                                   opts);
 288
 289                 readpage_iter_advance(&readpages_iter);
 290
 291                 rbio->bio.bi_iter.bi_sector = folio_sector(folio);
 292                 rbio->bio.bi_end_io = bch2_readpages_end_io;
 293                 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
 294
 295                 bchfs_read(trans, rbio, inode_inum(inode),
 296                            &readpages_iter);
 297                 bch2_trans_unlock(trans);
 298         }
 299
 300         bch2_pagecache_add_put(inode);
 301
 302         bch2_trans_put(trans);
 303         darray_exit(&readpages_iter.folios);
 304 }
 305
 306 static void bch2_read_single_folio_end_io(struct bio *bio)
 307 {
 308         complete(bio->bi_private);
 309 }
 310
 311 int bch2_read_single_folio(struct folio *folio, struct address_space *mapping)
 312 {
 313         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 314         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 315         struct bch_read_bio *rbio;
 316         struct bch_io_opts opts;
 317         int ret;
 318         DECLARE_COMPLETION_ONSTACK(done);
 319
 320         if (!bch2_folio_create(folio, GFP_KERNEL))
 321                 return -ENOMEM;
 322
 323         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
 324
 325         rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read),
 326                          opts);
 327         rbio->bio.bi_private = &done;
 328         rbio->bio.bi_end_io = bch2_read_single_folio_end_io;
 329
 330         rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC;
 331         rbio->bio.bi_iter.bi_sector = folio_sector(folio);
 332         BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
 333
 334         bch2_trans_run(c, (bchfs_read(trans, rbio, inode_inum(inode), NULL), 0));
 335         wait_for_completion(&done);
 336
 337         ret = blk_status_to_errno(rbio->bio.bi_status);
 338         bio_put(&rbio->bio);
 339
 340         if (ret < 0)
 341                 return ret;
 342
 343         folio_mark_uptodate(folio);
 344         return 0;
 345 }
 346
 347 int bch2_read_folio(struct file *file, struct folio *folio)
 348 {
 349         int ret;
 350
 351         ret = bch2_read_single_folio(folio, folio->mapping);
 352         folio_unlock(folio);
 353         return bch2_err_class(ret);
 354 }
 355
 356 /* writepages: */
 357
 358 struct bch_writepage_io {
 359         struct bch_inode_info           *inode;
 360
 361         /* must be last: */
 362         struct bch_write_op             op;
 363 };
 364
 365 struct bch_writepage_state {
 366         struct bch_writepage_io *io;
 367         struct bch_io_opts      opts;
 368         struct bch_folio_sector *tmp;
 369         unsigned                tmp_sectors;
 370 };
 371
 372 static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c,
 373                                                                   struct bch_inode_info *inode)
 374 {
 375         struct bch_writepage_state ret = { 0 };
 376
 377         bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode);
 378         return ret;
 379 }
 380
 381 /*
 382  * Determine when a writepage io is full. We have to limit writepage bios to a
 383  * single page per bvec (i.e. 1MB with 4k pages) because that is the limit to
 384  * what the bounce path in bch2_write_extent() can handle. In theory we could
 385  * loosen this restriction for non-bounce I/O, but we don't have that context
 386  * here. Ideally, we can up this limit and make it configurable in the future
 387  * when the bounce path can be enhanced to accommodate larger source bios.
 388  */
 389 static inline bool bch_io_full(struct bch_writepage_io *io, unsigned len)
 390 {
 391         struct bio *bio = &io->op.wbio.bio;
 392         return bio_full(bio, len) ||
 393                 (bio->bi_iter.bi_size + len > BIO_MAX_VECS * PAGE_SIZE);
 394 }
 395
 396 static void bch2_writepage_io_done(struct bch_write_op *op)
 397 {
 398         struct bch_writepage_io *io =
 399                 container_of(op, struct bch_writepage_io, op);
 400         struct bch_fs *c = io->op.c;
 401         struct bio *bio = &io->op.wbio.bio;
 402         struct folio_iter fi;
 403         unsigned i;
 404
 405         if (io->op.error) {
 406                 set_bit(EI_INODE_ERROR, &io->inode->ei_flags);
 407
 408                 bio_for_each_folio_all(fi, bio) {
 409                         struct bch_folio *s;
 410
 411                         folio_set_error(fi.folio);
 412                         mapping_set_error(fi.folio->mapping, -EIO);
 413
 414                         s = __bch2_folio(fi.folio);
 415                         spin_lock(&s->lock);
 416                         for (i = 0; i < folio_sectors(fi.folio); i++)
 417                                 s->s[i].nr_replicas = 0;
 418                         spin_unlock(&s->lock);
 419                 }
 420         }
 421
 422         if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) {
 423                 bio_for_each_folio_all(fi, bio) {
 424                         struct bch_folio *s;
 425
 426                         s = __bch2_folio(fi.folio);
 427                         spin_lock(&s->lock);
 428                         for (i = 0; i < folio_sectors(fi.folio); i++)
 429                                 s->s[i].nr_replicas = 0;
 430                         spin_unlock(&s->lock);
 431                 }
 432         }
 433
 434         /*
 435          * racing with fallocate can cause us to add fewer sectors than
 436          * expected - but we shouldn't add more sectors than expected:
 437          */
 438         WARN_ON_ONCE(io->op.i_sectors_delta > 0);
 439
 440         /*
 441          * (error (due to going RO) halfway through a page can screw that up
 442          * slightly)
 443          * XXX wtf?
 444            BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS);
 445          */
 446
 447         /*
 448          * PageWriteback is effectively our ref on the inode - fixup i_blocks
 449          * before calling end_page_writeback:
 450          */
 451         bch2_i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta);
 452
 453         bio_for_each_folio_all(fi, bio) {
 454                 struct bch_folio *s = __bch2_folio(fi.folio);
 455
 456                 if (atomic_dec_and_test(&s->write_count))
 457                         folio_end_writeback(fi.folio);
 458         }
 459
 460         bio_put(&io->op.wbio.bio);
 461 }
 462
 463 static void bch2_writepage_do_io(struct bch_writepage_state *w)
 464 {
 465         struct bch_writepage_io *io = w->io;
 466
 467         w->io = NULL;
 468         closure_call(&io->op.cl, bch2_write, NULL, NULL);
 469 }
 470
 471 /*
 472  * Get a bch_writepage_io and add @page to it - appending to an existing one if
 473  * possible, else allocating a new one:
 474  */
 475 static void bch2_writepage_io_alloc(struct bch_fs *c,
 476                                     struct writeback_control *wbc,
 477                                     struct bch_writepage_state *w,
 478                                     struct bch_inode_info *inode,
 479                                     u64 sector,
 480                                     unsigned nr_replicas)
 481 {
 482         struct bch_write_op *op;
 483
 484         w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS,
 485                                               REQ_OP_WRITE,
 486                                               GFP_KERNEL,
 487                                               &c->writepage_bioset),
 488                              struct bch_writepage_io, op.wbio.bio);
 489
 490         w->io->inode            = inode;
 491         op                      = &w->io->op;
 492         bch2_write_op_init(op, c, w->opts);
 493         op->target              = w->opts.foreground_target;
 494         op->nr_replicas         = nr_replicas;
 495         op->res.nr_replicas     = nr_replicas;
 496         op->write_point         = writepoint_hashed(inode->ei_last_dirtied);
 497         op->subvol              = inode->ei_subvol;
 498         op->pos                 = POS(inode->v.i_ino, sector);
 499         op->end_io              = bch2_writepage_io_done;
 500         op->devs_need_flush     = &inode->ei_devs_need_flush;
 501         op->wbio.bio.bi_iter.bi_sector = sector;
 502         op->wbio.bio.bi_opf     = wbc_to_write_flags(wbc);
 503 }
 504
 505 static int __bch2_writepage(struct folio *folio,
 506                             struct writeback_control *wbc,
 507                             void *data)
 508 {
 509         struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
 510         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 511         struct bch_writepage_state *w = data;
 512         struct bch_folio *s;
 513         unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX;
 514         loff_t i_size = i_size_read(&inode->v);
 515         int ret;
 516
 517         EBUG_ON(!folio_test_uptodate(folio));
 518
 519         /* Is the folio fully inside i_size? */
 520         if (folio_end_pos(folio) <= i_size)
 521                 goto do_io;
 522
 523         /* Is the folio fully outside i_size? (truncate in progress) */
 524         if (folio_pos(folio) >= i_size) {
 525                 folio_unlock(folio);
 526                 return 0;
 527         }
 528
 529         /*
 530          * The folio straddles i_size.  It must be zeroed out on each and every
 531          * writepage invocation because it may be mmapped.  "A file is mapped
 532          * in multiples of the folio size.  For a file that is not a multiple of
 533          * the  folio size, the remaining memory is zeroed when mapped, and
 534          * writes to that region are not written out to the file."
 535          */
 536         folio_zero_segment(folio,
 537                            i_size - folio_pos(folio),
 538                            folio_size(folio));
 539 do_io:
 540         f_sectors = folio_sectors(folio);
 541         s = bch2_folio(folio);
 542
 543         if (f_sectors > w->tmp_sectors) {
 544                 kfree(w->tmp);
 545                 w->tmp = kcalloc(f_sectors, sizeof(struct bch_folio_sector), __GFP_NOFAIL);
 546                 w->tmp_sectors = f_sectors;
 547         }
 548
 549         /*
 550          * Things get really hairy with errors during writeback:
 551          */
 552         ret = bch2_get_folio_disk_reservation(c, inode, folio, false);
 553         BUG_ON(ret);
 554
 555         /* Before unlocking the page, get copy of reservations: */
 556         spin_lock(&s->lock);
 557         memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors);
 558
 559         for (i = 0; i < f_sectors; i++) {
 560                 if (s->s[i].state < SECTOR_dirty)
 561                         continue;
 562
 563                 nr_replicas_this_write =
 564                         min_t(unsigned, nr_replicas_this_write,
 565                               s->s[i].nr_replicas +
 566                               s->s[i].replicas_reserved);
 567         }
 568
 569         for (i = 0; i < f_sectors; i++) {
 570                 if (s->s[i].state < SECTOR_dirty)
 571                         continue;
 572
 573                 s->s[i].nr_replicas = w->opts.compression
 574                         ? 0 : nr_replicas_this_write;
 575
 576                 s->s[i].replicas_reserved = 0;
 577                 bch2_folio_sector_set(folio, s, i, SECTOR_allocated);
 578         }
 579         spin_unlock(&s->lock);
 580
 581         BUG_ON(atomic_read(&s->write_count));
 582         atomic_set(&s->write_count, 1);
 583
 584         BUG_ON(folio_test_writeback(folio));
 585         folio_start_writeback(folio);
 586
 587         folio_unlock(folio);
 588
 589         offset = 0;
 590         while (1) {
 591                 unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0;
 592                 u64 sector;
 593
 594                 while (offset < f_sectors &&
 595                        w->tmp[offset].state < SECTOR_dirty)
 596                         offset++;
 597
 598                 if (offset == f_sectors)
 599                         break;
 600
 601                 while (offset + sectors < f_sectors &&
 602                        w->tmp[offset + sectors].state >= SECTOR_dirty) {
 603                         reserved_sectors += w->tmp[offset + sectors].replicas_reserved;
 604                         dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty;
 605                         sectors++;
 606                 }
 607                 BUG_ON(!sectors);
 608
 609                 sector = folio_sector(folio) + offset;
 610
 611                 if (w->io &&
 612                     (w->io->op.res.nr_replicas != nr_replicas_this_write ||
 613                      bch_io_full(w->io, sectors << 9) ||
 614                      bio_end_sector(&w->io->op.wbio.bio) != sector))
 615                         bch2_writepage_do_io(w);
 616
 617                 if (!w->io)
 618                         bch2_writepage_io_alloc(c, wbc, w, inode, sector,
 619                                                 nr_replicas_this_write);
 620
 621                 atomic_inc(&s->write_count);
 622
 623                 BUG_ON(inode != w->io->inode);
 624                 BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio,
 625                                      sectors << 9, offset << 9));
 626
 627                 /* Check for writing past i_size: */
 628                 WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) >
 629                           round_up(i_size, block_bytes(c)) &&
 630                           !test_bit(BCH_FS_emergency_ro, &c->flags),
 631                           "writing past i_size: %llu > %llu (unrounded %llu)\n",
 632                           bio_end_sector(&w->io->op.wbio.bio) << 9,
 633                           round_up(i_size, block_bytes(c)),
 634                           i_size);
 635
 636                 w->io->op.res.sectors += reserved_sectors;
 637                 w->io->op.i_sectors_delta -= dirty_sectors;
 638                 w->io->op.new_i_size = i_size;
 639
 640                 offset += sectors;
 641         }
 642
 643         if (atomic_dec_and_test(&s->write_count))
 644                 folio_end_writeback(folio);
 645
 646         return 0;
 647 }
 648
 649 int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc)
 650 {
 651         struct bch_fs *c = mapping->host->i_sb->s_fs_info;
 652         struct bch_writepage_state w =
 653                 bch_writepage_state_init(c, to_bch_ei(mapping->host));
 654         struct blk_plug plug;
 655         int ret;
 656
 657         blk_start_plug(&plug);
 658         ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w);
 659         if (w.io)
 660                 bch2_writepage_do_io(&w);
 661         blk_finish_plug(&plug);
 662         kfree(w.tmp);
 663         return bch2_err_class(ret);
 664 }
 665
 666 /* buffered writes: */
 667
 668 int bch2_write_begin(struct file *file, struct address_space *mapping,
 669                      loff_t pos, unsigned len,
 670                      struct page **pagep, void **fsdata)
 671 {
 672         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 673         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 674         struct bch2_folio_reservation *res;
 675         struct folio *folio;
 676         unsigned offset;
 677         int ret = -ENOMEM;
 678
 679         res = kmalloc(sizeof(*res), GFP_KERNEL);
 680         if (!res)
 681                 return -ENOMEM;
 682
 683         bch2_folio_reservation_init(c, inode, res);
 684         *fsdata = res;
 685
 686         bch2_pagecache_add_get(inode);
 687
 688         folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT,
 689                                 FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE,
 690                                 mapping_gfp_mask(mapping));
 691         if (IS_ERR_OR_NULL(folio))
 692                 goto err_unlock;
 693
 694         offset = pos - folio_pos(folio);
 695         len = min_t(size_t, len, folio_end_pos(folio) - pos);
 696
 697         if (folio_test_uptodate(folio))
 698                 goto out;
 699
 700         /* If we're writing entire folio, don't need to read it in first: */
 701         if (!offset && len == folio_size(folio))
 702                 goto out;
 703
 704         if (!offset && pos + len >= inode->v.i_size) {
 705                 folio_zero_segment(folio, len, folio_size(folio));
 706                 flush_dcache_folio(folio);
 707                 goto out;
 708         }
 709
 710         if (folio_pos(folio) >= inode->v.i_size) {
 711                 folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio));
 712                 flush_dcache_folio(folio);
 713                 goto out;
 714         }
 715 readpage:
 716         ret = bch2_read_single_folio(folio, mapping);
 717         if (ret)
 718                 goto err;
 719 out:
 720         ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);
 721         if (ret)
 722                 goto err;
 723
 724         ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len);
 725         if (ret) {
 726                 if (!folio_test_uptodate(folio)) {
 727                         /*
 728                          * If the folio hasn't been read in, we won't know if we
 729                          * actually need a reservation - we don't actually need
 730                          * to read here, we just need to check if the folio is
 731                          * fully backed by uncompressed data:
 732                          */
 733                         goto readpage;
 734                 }
 735
 736                 goto err;
 737         }
 738
 739         *pagep = &folio->page;
 740         return 0;
 741 err:
 742         folio_unlock(folio);
 743         folio_put(folio);
 744         *pagep = NULL;
 745 err_unlock:
 746         bch2_pagecache_add_put(inode);
 747         kfree(res);
 748         *fsdata = NULL;
 749         return bch2_err_class(ret);
 750 }
 751
 752 int bch2_write_end(struct file *file, struct address_space *mapping,
 753                    loff_t pos, unsigned len, unsigned copied,
 754                    struct page *page, void *fsdata)
 755 {
 756         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 757         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 758         struct bch2_folio_reservation *res = fsdata;
 759         struct folio *folio = page_folio(page);
 760         unsigned offset = pos - folio_pos(folio);
 761
 762         lockdep_assert_held(&inode->v.i_rwsem);
 763         BUG_ON(offset + copied > folio_size(folio));
 764
 765         if (unlikely(copied < len && !folio_test_uptodate(folio))) {
 766                 /*
 767                  * The folio needs to be read in, but that would destroy
 768                  * our partial write - simplest thing is to just force
 769                  * userspace to redo the write:
 770                  */
 771                 folio_zero_range(folio, 0, folio_size(folio));
 772                 flush_dcache_folio(folio);
 773                 copied = 0;
 774         }
 775
 776         spin_lock(&inode->v.i_lock);
 777         if (pos + copied > inode->v.i_size)
 778                 i_size_write(&inode->v, pos + copied);
 779         spin_unlock(&inode->v.i_lock);
 780
 781         if (copied) {
 782                 if (!folio_test_uptodate(folio))
 783                         folio_mark_uptodate(folio);
 784
 785                 bch2_set_folio_dirty(c, inode, folio, res, offset, copied);
 786
 787                 inode->ei_last_dirtied = (unsigned long) current;
 788         }
 789
 790         folio_unlock(folio);
 791         folio_put(folio);
 792         bch2_pagecache_add_put(inode);
 793
 794         bch2_folio_reservation_put(c, inode, res);
 795         kfree(res);
 796
 797         return copied;
 798 }
 799
 800 static noinline void folios_trunc(folios *fs, struct folio **fi)
 801 {
 802         while (fs->data + fs->nr > fi) {
 803                 struct folio *f = darray_pop(fs);
 804
 805                 folio_unlock(f);
 806                 folio_put(f);
 807         }
 808 }
 809
 810 static int __bch2_buffered_write(struct bch_inode_info *inode,
 811                                  struct address_space *mapping,
 812                                  struct iov_iter *iter,
 813                                  loff_t pos, unsigned len,
 814                                  bool inode_locked)
 815 {
 816         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 817         struct bch2_folio_reservation res;
 818         folios fs;
 819         struct folio *f;
 820         unsigned copied = 0, f_offset, f_copied;
 821         u64 end = pos + len, f_pos, f_len;
 822         loff_t last_folio_pos = inode->v.i_size;
 823         int ret = 0;
 824
 825         BUG_ON(!len);
 826
 827         bch2_folio_reservation_init(c, inode, &res);
 828         darray_init(&fs);
 829
 830         ret = bch2_filemap_get_contig_folios_d(mapping, pos, end,
 831                                    FGP_LOCK|FGP_WRITE|FGP_STABLE|FGP_CREAT,
 832                                    mapping_gfp_mask(mapping),
 833                                    &fs);
 834         if (ret)
 835                 goto out;
 836
 837         BUG_ON(!fs.nr);
 838
 839         /*
 840          * If we're not using the inode lock, we need to lock all the folios for
 841          * atomiticity of writes vs. other writes:
 842          */
 843         if (!inode_locked && folio_end_pos(darray_last(fs)) < end) {
 844                 ret = -BCH_ERR_need_inode_lock;
 845                 goto out;
 846         }
 847
 848         f = darray_first(fs);
 849         if (pos != folio_pos(f) && !folio_test_uptodate(f)) {
 850                 ret = bch2_read_single_folio(f, mapping);
 851                 if (ret)
 852                         goto out;
 853         }
 854
 855         f = darray_last(fs);
 856         end = min(end, folio_end_pos(f));
 857         last_folio_pos = folio_pos(f);
 858         if (end != folio_end_pos(f) && !folio_test_uptodate(f)) {
 859                 if (end >= inode->v.i_size) {
 860                         folio_zero_range(f, 0, folio_size(f));
 861                 } else {
 862                         ret = bch2_read_single_folio(f, mapping);
 863                         if (ret)
 864                                 goto out;
 865                 }
 866         }
 867
 868         ret = bch2_folio_set(c, inode_inum(inode), fs.data, fs.nr);
 869         if (ret)
 870                 goto out;
 871
 872         f_pos = pos;
 873         f_offset = pos - folio_pos(darray_first(fs));
 874         darray_for_each(fs, fi) {
 875                 f = *fi;
 876                 f_len = min(end, folio_end_pos(f)) - f_pos;
 877
 878                 /*
 879                  * XXX: per POSIX and fstests generic/275, on -ENOSPC we're
 880                  * supposed to write as much as we have disk space for.
 881                  *
 882                  * On failure here we should still write out a partial page if
 883                  * we aren't completely out of disk space - we don't do that
 884                  * yet:
 885                  */
 886                 ret = bch2_folio_reservation_get(c, inode, f, &res, f_offset, f_len);
 887                 if (unlikely(ret)) {
 888                         folios_trunc(&fs, fi);
 889                         if (!fs.nr)
 890                                 goto out;
 891
 892                         end = min(end, folio_end_pos(darray_last(fs)));
 893                         break;
 894                 }
 895
 896                 f_pos = folio_end_pos(f);
 897                 f_offset = 0;
 898         }
 899
 900         if (mapping_writably_mapped(mapping))
 901                 darray_for_each(fs, fi)
 902                         flush_dcache_folio(*fi);
 903
 904         f_pos = pos;
 905         f_offset = pos - folio_pos(darray_first(fs));
 906         darray_for_each(fs, fi) {
 907                 f = *fi;
 908                 f_len = min(end, folio_end_pos(f)) - f_pos;
 909                 f_copied = copy_page_from_iter_atomic(&f->page, f_offset, f_len, iter);
 910                 if (!f_copied) {
 911                         folios_trunc(&fs, fi);
 912                         break;
 913                 }
 914
 915                 if (!folio_test_uptodate(f) &&
 916                     f_copied != folio_size(f) &&
 917                     pos + copied + f_copied < inode->v.i_size) {
 918                         iov_iter_revert(iter, f_copied);
 919                         folio_zero_range(f, 0, folio_size(f));
 920                         folios_trunc(&fs, fi);
 921                         break;
 922                 }
 923
 924                 flush_dcache_folio(f);
 925                 copied += f_copied;
 926
 927                 if (f_copied != f_len) {
 928                         folios_trunc(&fs, fi + 1);
 929                         break;
 930                 }
 931
 932                 f_pos = folio_end_pos(f);
 933                 f_offset = 0;
 934         }
 935
 936         if (!copied)
 937                 goto out;
 938
 939         end = pos + copied;
 940
 941         spin_lock(&inode->v.i_lock);
 942         if (end > inode->v.i_size) {
 943                 BUG_ON(!inode_locked);
 944                 i_size_write(&inode->v, end);
 945         }
 946         spin_unlock(&inode->v.i_lock);
 947
 948         f_pos = pos;
 949         f_offset = pos - folio_pos(darray_first(fs));
 950         darray_for_each(fs, fi) {
 951                 f = *fi;
 952                 f_len = min(end, folio_end_pos(f)) - f_pos;
 953
 954                 if (!folio_test_uptodate(f))
 955                         folio_mark_uptodate(f);
 956
 957                 bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len);
 958
 959                 f_pos = folio_end_pos(f);
 960                 f_offset = 0;
 961         }
 962
 963         inode->ei_last_dirtied = (unsigned long) current;
 964 out:
 965         darray_for_each(fs, fi) {
 966                 folio_unlock(*fi);
 967                 folio_put(*fi);
 968         }
 969
 970         /*
 971          * If the last folio added to the mapping starts beyond current EOF, we
 972          * performed a short write but left around at least one post-EOF folio.
 973          * Clean up the mapping before we return.
 974          */
 975         if (last_folio_pos >= inode->v.i_size)
 976                 truncate_pagecache(&inode->v, inode->v.i_size);
 977
 978         darray_exit(&fs);
 979         bch2_folio_reservation_put(c, inode, &res);
 980
 981         return copied ?: ret;
 982 }
 983
 984 static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
 985 {
 986         struct file *file = iocb->ki_filp;
 987         struct address_space *mapping = file->f_mapping;
 988         struct bch_inode_info *inode = file_bch_inode(file);
 989         loff_t pos;
 990         bool inode_locked = false;
 991         ssize_t written = 0, written2 = 0, ret = 0;
 992
 993         /*
 994          * We don't take the inode lock unless i_size will be changing. Folio
 995          * locks provide exclusion with other writes, and the pagecache add lock
 996          * provides exclusion with truncate and hole punching.
 997          *
 998          * There is one nasty corner case where atomicity would be broken
 999          * without great care: when copying data from userspace to the page
1000          * cache, we do that with faults disable - a page fault would recurse
1001          * back into the filesystem, taking filesystem locks again, and
1002          * deadlock; so it's done with faults disabled, and we fault in the user
1003          * buffer when we aren't holding locks.
1004          *
1005          * If we do part of the write, but we then race and in the userspace
1006          * buffer have been evicted and are no longer resident, then we have to
1007          * drop our folio locks to re-fault them in, breaking write atomicity.
1008          *
1009          * To fix this, we restart the write from the start, if we weren't
1010          * holding the inode lock.
1011          *
1012          * There is another wrinkle after that; if we restart the write from the
1013          * start, and then get an unrecoverable error, we _cannot_ claim to
1014          * userspace that we did not write data we actually did - so we must
1015          * track (written2) the most we ever wrote.
1016          */
1017
1018         if ((iocb->ki_flags & IOCB_APPEND) ||
1019             (iocb->ki_pos + iov_iter_count(iter) > i_size_read(&inode->v))) {
1020                 inode_lock(&inode->v);
1021                 inode_locked = true;
1022         }
1023
1024         ret = generic_write_checks(iocb, iter);
1025         if (ret <= 0)
1026                 goto unlock;
1027
1028         ret = file_remove_privs_flags(file, !inode_locked ? IOCB_NOWAIT : 0);
1029         if (ret) {
1030                 if (!inode_locked) {
1031                         inode_lock(&inode->v);
1032                         inode_locked = true;
1033                         ret = file_remove_privs_flags(file, 0);
1034                 }
1035                 if (ret)
1036                         goto unlock;
1037         }
1038
1039         ret = file_update_time(file);
1040         if (ret)
1041                 goto unlock;
1042
1043         pos = iocb->ki_pos;
1044
1045         bch2_pagecache_add_get(inode);
1046
1047         if (!inode_locked &&
1048             (iocb->ki_pos + iov_iter_count(iter) > i_size_read(&inode->v)))
1049                 goto get_inode_lock;
1050
1051         do {
1052                 unsigned offset = pos & (PAGE_SIZE - 1);
1053                 unsigned bytes = iov_iter_count(iter);
1054 again:
1055                 /*
1056                  * Bring in the user page that we will copy from _first_.
1057                  * Otherwise there's a nasty deadlock on copying from the
1058                  * same page as we're writing to, without it being marked
1059                  * up-to-date.
1060                  *
1061                  * Not only is this an optimisation, but it is also required
1062                  * to check that the address is actually valid, when atomic
1063                  * usercopies are used, below.
1064                  */
1065                 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1066                         bytes = min_t(unsigned long, iov_iter_count(iter),
1067                                       PAGE_SIZE - offset);
1068
1069                         if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1070                                 ret = -EFAULT;
1071                                 break;
1072                         }
1073                 }
1074
1075                 if (unlikely(bytes != iov_iter_count(iter) && !inode_locked))
1076                         goto get_inode_lock;
1077
1078                 if (unlikely(fatal_signal_pending(current))) {
1079                         ret = -EINTR;
1080                         break;
1081                 }
1082
1083                 ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes, inode_locked);
1084                 if (ret == -BCH_ERR_need_inode_lock)
1085                         goto get_inode_lock;
1086                 if (unlikely(ret < 0))
1087                         break;
1088
1089                 cond_resched();
1090
1091                 if (unlikely(ret == 0)) {
1092                         /*
1093                          * If we were unable to copy any data at all, we must
1094                          * fall back to a single segment length write.
1095                          *
1096                          * If we didn't fallback here, we could livelock
1097                          * because not all segments in the iov can be copied at
1098                          * once without a pagefault.
1099                          */
1100                         bytes = min_t(unsigned long, PAGE_SIZE - offset,
1101                                       iov_iter_single_seg_count(iter));
1102                         goto again;
1103                 }
1104                 pos += ret;
1105                 written += ret;
1106                 written2 = max(written, written2);
1107
1108                 if (ret != bytes && !inode_locked)
1109                         goto get_inode_lock;
1110                 ret = 0;
1111
1112                 balance_dirty_pages_ratelimited(mapping);
1113
1114                 if (0) {
1115 get_inode_lock:
1116                         bch2_pagecache_add_put(inode);
1117                         inode_lock(&inode->v);
1118                         inode_locked = true;
1119                         bch2_pagecache_add_get(inode);
1120
1121                         iov_iter_revert(iter, written);
1122                         pos -= written;
1123                         written = 0;
1124                         ret = 0;
1125                 }
1126         } while (iov_iter_count(iter));
1127         bch2_pagecache_add_put(inode);
1128 unlock:
1129         if (inode_locked)
1130                 inode_unlock(&inode->v);
1131
1132         iocb->ki_pos += written;
1133
1134         ret = max(written, written2) ?: ret;
1135         if (ret > 0)
1136                 ret = generic_write_sync(iocb, ret);
1137         return ret;
1138 }
1139
1140 ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *iter)
1141 {
1142         ssize_t ret = iocb->ki_flags & IOCB_DIRECT
1143                 ? bch2_direct_write(iocb, iter)
1144                 : bch2_buffered_write(iocb, iter);
1145
1146         return bch2_err_class(ret);
1147 }
1148
1149 void bch2_fs_fs_io_buffered_exit(struct bch_fs *c)
1150 {
1151         bioset_exit(&c->writepage_bioset);
1152 }
1153
1154 int bch2_fs_fs_io_buffered_init(struct bch_fs *c)
1155 {
1156         if (bioset_init(&c->writepage_bioset,
1157                         4, offsetof(struct bch_writepage_io, op.wbio.bio),
1158                         BIOSET_NEED_BVECS))
1159                 return -BCH_ERR_ENOMEM_writepage_bioset_init;
1160
1161         return 0;
1162 }
1163
1164 #endif /* NO_BCACHEFS_FS */