fs/btrfs/file.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/fs.h>
   7 #include <linux/pagemap.h>
   8 #include <linux/time.h>
   9 #include <linux/init.h>
  10 #include <linux/string.h>
  11 #include <linux/backing-dev.h>
  12 #include <linux/falloc.h>
  13 #include <linux/writeback.h>
  14 #include <linux/compat.h>
  15 #include <linux/slab.h>
  16 #include <linux/btrfs.h>
  17 #include <linux/uio.h>
  18 #include <linux/iversion.h>
  19 #include "ctree.h"
  20 #include "disk-io.h"
  21 #include "transaction.h"
  22 #include "btrfs_inode.h"
  23 #include "print-tree.h"
  24 #include "tree-log.h"
  25 #include "locking.h"
  26 #include "volumes.h"
  27 #include "qgroup.h"
  28 #include "compression.h"
  29 #include "delalloc-space.h"
  30
  31 static struct kmem_cache *btrfs_inode_defrag_cachep;
  32 /*
  33  * when auto defrag is enabled we
  34  * queue up these defrag structs to remember which
  35  * inodes need defragging passes
  36  */
  37 struct inode_defrag {
  38         struct rb_node rb_node;
  39         /* objectid */
  40         u64 ino;
  41         /*
  42          * transid where the defrag was added, we search for
  43          * extents newer than this
  44          */
  45         u64 transid;
  46
  47         /* root objectid */
  48         u64 root;
  49
  50         /* last offset we were able to defrag */
  51         u64 last_offset;
  52
  53         /* if we've wrapped around back to zero once already */
  54         int cycled;
  55 };
  56
  57 static int __compare_inode_defrag(struct inode_defrag *defrag1,
  58                                   struct inode_defrag *defrag2)
  59 {
  60         if (defrag1->root > defrag2->root)
  61                 return 1;
  62         else if (defrag1->root < defrag2->root)
  63                 return -1;
  64         else if (defrag1->ino > defrag2->ino)
  65                 return 1;
  66         else if (defrag1->ino < defrag2->ino)
  67                 return -1;
  68         else
  69                 return 0;
  70 }
  71
  72 /* pop a record for an inode into the defrag tree.  The lock
  73  * must be held already
  74  *
  75  * If you're inserting a record for an older transid than an
  76  * existing record, the transid already in the tree is lowered
  77  *
  78  * If an existing record is found the defrag item you
  79  * pass in is freed
  80  */
  81 static int __btrfs_add_inode_defrag(struct btrfs_inode *inode,
  82                                     struct inode_defrag *defrag)
  83 {
  84         struct btrfs_fs_info *fs_info = inode->root->fs_info;
  85         struct inode_defrag *entry;
  86         struct rb_node **p;
  87         struct rb_node *parent = NULL;
  88         int ret;
  89
  90         p = &fs_info->defrag_inodes.rb_node;
  91         while (*p) {
  92                 parent = *p;
  93                 entry = rb_entry(parent, struct inode_defrag, rb_node);
  94
  95                 ret = __compare_inode_defrag(defrag, entry);
  96                 if (ret < 0)
  97                         p = &parent->rb_left;
  98                 else if (ret > 0)
  99                         p = &parent->rb_right;
 100                 else {
 101                         /* if we're reinserting an entry for
 102                          * an old defrag run, make sure to
 103                          * lower the transid of our existing record
 104                          */
 105                         if (defrag->transid < entry->transid)
 106                                 entry->transid = defrag->transid;
 107                         if (defrag->last_offset > entry->last_offset)
 108                                 entry->last_offset = defrag->last_offset;
 109                         return -EEXIST;
 110                 }
 111         }
 112         set_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags);
 113         rb_link_node(&defrag->rb_node, parent, p);
 114         rb_insert_color(&defrag->rb_node, &fs_info->defrag_inodes);
 115         return 0;
 116 }
 117
 118 static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info)
 119 {
 120         if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
 121                 return 0;
 122
 123         if (btrfs_fs_closing(fs_info))
 124                 return 0;
 125
 126         return 1;
 127 }
 128
 129 /*
 130  * insert a defrag record for this inode if auto defrag is
 131  * enabled
 132  */
 133 int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
 134                            struct btrfs_inode *inode)
 135 {
 136         struct btrfs_root *root = inode->root;
 137         struct btrfs_fs_info *fs_info = root->fs_info;
 138         struct inode_defrag *defrag;
 139         u64 transid;
 140         int ret;
 141
 142         if (!__need_auto_defrag(fs_info))
 143                 return 0;
 144
 145         if (test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags))
 146                 return 0;
 147
 148         if (trans)
 149                 transid = trans->transid;
 150         else
 151                 transid = inode->root->last_trans;
 152
 153         defrag = kmem_cache_zalloc(btrfs_inode_defrag_cachep, GFP_NOFS);
 154         if (!defrag)
 155                 return -ENOMEM;
 156
 157         defrag->ino = btrfs_ino(inode);
 158         defrag->transid = transid;
 159         defrag->root = root->root_key.objectid;
 160
 161         spin_lock(&fs_info->defrag_inodes_lock);
 162         if (!test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags)) {
 163                 /*
 164                  * If we set IN_DEFRAG flag and evict the inode from memory,
 165                  * and then re-read this inode, this new inode doesn't have
 166                  * IN_DEFRAG flag. At the case, we may find the existed defrag.
 167                  */
 168                 ret = __btrfs_add_inode_defrag(inode, defrag);
 169                 if (ret)
 170                         kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 171         } else {
 172                 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 173         }
 174         spin_unlock(&fs_info->defrag_inodes_lock);
 175         return 0;
 176 }
 177
 178 /*
 179  * Requeue the defrag object. If there is a defrag object that points to
 180  * the same inode in the tree, we will merge them together (by
 181  * __btrfs_add_inode_defrag()) and free the one that we want to requeue.
 182  */
 183 static void btrfs_requeue_inode_defrag(struct btrfs_inode *inode,
 184                                        struct inode_defrag *defrag)
 185 {
 186         struct btrfs_fs_info *fs_info = inode->root->fs_info;
 187         int ret;
 188
 189         if (!__need_auto_defrag(fs_info))
 190                 goto out;
 191
 192         /*
 193          * Here we don't check the IN_DEFRAG flag, because we need merge
 194          * them together.
 195          */
 196         spin_lock(&fs_info->defrag_inodes_lock);
 197         ret = __btrfs_add_inode_defrag(inode, defrag);
 198         spin_unlock(&fs_info->defrag_inodes_lock);
 199         if (ret)
 200                 goto out;
 201         return;
 202 out:
 203         kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 204 }
 205
 206 /*
 207  * pick the defragable inode that we want, if it doesn't exist, we will get
 208  * the next one.
 209  */
 210 static struct inode_defrag *
 211 btrfs_pick_defrag_inode(struct btrfs_fs_info *fs_info, u64 root, u64 ino)
 212 {
 213         struct inode_defrag *entry = NULL;
 214         struct inode_defrag tmp;
 215         struct rb_node *p;
 216         struct rb_node *parent = NULL;
 217         int ret;
 218
 219         tmp.ino = ino;
 220         tmp.root = root;
 221
 222         spin_lock(&fs_info->defrag_inodes_lock);
 223         p = fs_info->defrag_inodes.rb_node;
 224         while (p) {
 225                 parent = p;
 226                 entry = rb_entry(parent, struct inode_defrag, rb_node);
 227
 228                 ret = __compare_inode_defrag(&tmp, entry);
 229                 if (ret < 0)
 230                         p = parent->rb_left;
 231                 else if (ret > 0)
 232                         p = parent->rb_right;
 233                 else
 234                         goto out;
 235         }
 236
 237         if (parent && __compare_inode_defrag(&tmp, entry) > 0) {
 238                 parent = rb_next(parent);
 239                 if (parent)
 240                         entry = rb_entry(parent, struct inode_defrag, rb_node);
 241                 else
 242                         entry = NULL;
 243         }
 244 out:
 245         if (entry)
 246                 rb_erase(parent, &fs_info->defrag_inodes);
 247         spin_unlock(&fs_info->defrag_inodes_lock);
 248         return entry;
 249 }
 250
 251 void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info)
 252 {
 253         struct inode_defrag *defrag;
 254         struct rb_node *node;
 255
 256         spin_lock(&fs_info->defrag_inodes_lock);
 257         node = rb_first(&fs_info->defrag_inodes);
 258         while (node) {
 259                 rb_erase(node, &fs_info->defrag_inodes);
 260                 defrag = rb_entry(node, struct inode_defrag, rb_node);
 261                 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 262
 263                 cond_resched_lock(&fs_info->defrag_inodes_lock);
 264
 265                 node = rb_first(&fs_info->defrag_inodes);
 266         }
 267         spin_unlock(&fs_info->defrag_inodes_lock);
 268 }
 269
 270 #define BTRFS_DEFRAG_BATCH      1024
 271
 272 static int __btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info,
 273                                     struct inode_defrag *defrag)
 274 {
 275         struct btrfs_root *inode_root;
 276         struct inode *inode;
 277         struct btrfs_key key;
 278         struct btrfs_ioctl_defrag_range_args range;
 279         int num_defrag;
 280         int index;
 281         int ret;
 282
 283         /* get the inode */
 284         key.objectid = defrag->root;
 285         key.type = BTRFS_ROOT_ITEM_KEY;
 286         key.offset = (u64)-1;
 287
 288         index = srcu_read_lock(&fs_info->subvol_srcu);
 289
 290         inode_root = btrfs_read_fs_root_no_name(fs_info, &key);
 291         if (IS_ERR(inode_root)) {
 292                 ret = PTR_ERR(inode_root);
 293                 goto cleanup;
 294         }
 295
 296         key.objectid = defrag->ino;
 297         key.type = BTRFS_INODE_ITEM_KEY;
 298         key.offset = 0;
 299         inode = btrfs_iget(fs_info->sb, &key, inode_root, NULL);
 300         if (IS_ERR(inode)) {
 301                 ret = PTR_ERR(inode);
 302                 goto cleanup;
 303         }
 304         srcu_read_unlock(&fs_info->subvol_srcu, index);
 305
 306         /* do a chunk of defrag */
 307         clear_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags);
 308         memset(&range, 0, sizeof(range));
 309         range.len = (u64)-1;
 310         range.start = defrag->last_offset;
 311
 312         sb_start_write(fs_info->sb);
 313         num_defrag = btrfs_defrag_file(inode, NULL, &range, defrag->transid,
 314                                        BTRFS_DEFRAG_BATCH);
 315         sb_end_write(fs_info->sb);
 316         /*
 317          * if we filled the whole defrag batch, there
 318          * must be more work to do.  Queue this defrag
 319          * again
 320          */
 321         if (num_defrag == BTRFS_DEFRAG_BATCH) {
 322                 defrag->last_offset = range.start;
 323                 btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
 324         } else if (defrag->last_offset && !defrag->cycled) {
 325                 /*
 326                  * we didn't fill our defrag batch, but
 327                  * we didn't start at zero.  Make sure we loop
 328                  * around to the start of the file.
 329                  */
 330                 defrag->last_offset = 0;
 331                 defrag->cycled = 1;
 332                 btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
 333         } else {
 334                 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 335         }
 336
 337         iput(inode);
 338         return 0;
 339 cleanup:
 340         srcu_read_unlock(&fs_info->subvol_srcu, index);
 341         kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 342         return ret;
 343 }
 344
 345 /*
 346  * run through the list of inodes in the FS that need
 347  * defragging
 348  */
 349 int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
 350 {
 351         struct inode_defrag *defrag;
 352         u64 first_ino = 0;
 353         u64 root_objectid = 0;
 354
 355         atomic_inc(&fs_info->defrag_running);
 356         while (1) {
 357                 /* Pause the auto defragger. */
 358                 if (test_bit(BTRFS_FS_STATE_REMOUNTING,
 359                              &fs_info->fs_state))
 360                         break;
 361
 362                 if (!__need_auto_defrag(fs_info))
 363                         break;
 364
 365                 /* find an inode to defrag */
 366                 defrag = btrfs_pick_defrag_inode(fs_info, root_objectid,
 367                                                  first_ino);
 368                 if (!defrag) {
 369                         if (root_objectid || first_ino) {
 370                                 root_objectid = 0;
 371                                 first_ino = 0;
 372                                 continue;
 373                         } else {
 374                                 break;
 375                         }
 376                 }
 377
 378                 first_ino = defrag->ino + 1;
 379                 root_objectid = defrag->root;
 380
 381                 __btrfs_run_defrag_inode(fs_info, defrag);
 382         }
 383         atomic_dec(&fs_info->defrag_running);
 384
 385         /*
 386          * during unmount, we use the transaction_wait queue to
 387          * wait for the defragger to stop
 388          */
 389         wake_up(&fs_info->transaction_wait);
 390         return 0;
 391 }
 392
 393 /* simple helper to fault in pages and copy.  This should go away
 394  * and be replaced with calls into generic code.
 395  */
 396 static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
 397                                          struct page **prepared_pages,
 398                                          struct iov_iter *i)
 399 {
 400         size_t copied = 0;
 401         size_t total_copied = 0;
 402         int pg = 0;
 403         int offset = offset_in_page(pos);
 404
 405         while (write_bytes > 0) {
 406                 size_t count = min_t(size_t,
 407                                      PAGE_SIZE - offset, write_bytes);
 408                 struct page *page = prepared_pages[pg];
 409                 /*
 410                  * Copy data from userspace to the current page
 411                  */
 412                 copied = iov_iter_copy_from_user_atomic(page, i, offset, count);
 413
 414                 /* Flush processor's dcache for this page */
 415                 flush_dcache_page(page);
 416
 417                 /*
 418                  * if we get a partial write, we can end up with
 419                  * partially up to date pages.  These add
 420                  * a lot of complexity, so make sure they don't
 421                  * happen by forcing this copy to be retried.
 422                  *
 423                  * The rest of the btrfs_file_write code will fall
 424                  * back to page at a time copies after we return 0.
 425                  */
 426                 if (!PageUptodate(page) && copied < count)
 427                         copied = 0;
 428
 429                 iov_iter_advance(i, copied);
 430                 write_bytes -= copied;
 431                 total_copied += copied;
 432
 433                 /* Return to btrfs_file_write_iter to fault page */
 434                 if (unlikely(copied == 0))
 435                         break;
 436
 437                 if (copied < PAGE_SIZE - offset) {
 438                         offset += copied;
 439                 } else {
 440                         pg++;
 441                         offset = 0;
 442                 }
 443         }
 444         return total_copied;
 445 }
 446
 447 /*
 448  * unlocks pages after btrfs_file_write is done with them
 449  */
 450 static void btrfs_drop_pages(struct page **pages, size_t num_pages)
 451 {
 452         size_t i;
 453         for (i = 0; i < num_pages; i++) {
 454                 /* page checked is some magic around finding pages that
 455                  * have been modified without going through btrfs_set_page_dirty
 456                  * clear it here. There should be no need to mark the pages
 457                  * accessed as prepare_pages should have marked them accessed
 458                  * in prepare_pages via find_or_create_page()
 459                  */
 460                 ClearPageChecked(pages[i]);
 461                 unlock_page(pages[i]);
 462                 put_page(pages[i]);
 463         }
 464 }
 465
 466 static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode,
 467                                          const u64 start,
 468                                          const u64 len,
 469                                          struct extent_state **cached_state)
 470 {
 471         u64 search_start = start;
 472         const u64 end = start + len - 1;
 473
 474         while (search_start < end) {
 475                 const u64 search_len = end - search_start + 1;
 476                 struct extent_map *em;
 477                 u64 em_len;
 478                 int ret = 0;
 479
 480                 em = btrfs_get_extent(inode, NULL, 0, search_start,
 481                                       search_len, 0);
 482                 if (IS_ERR(em))
 483                         return PTR_ERR(em);
 484
 485                 if (em->block_start != EXTENT_MAP_HOLE)
 486                         goto next;
 487
 488                 em_len = em->len;
 489                 if (em->start < search_start)
 490                         em_len -= search_start - em->start;
 491                 if (em_len > search_len)
 492                         em_len = search_len;
 493
 494                 ret = set_extent_bit(&inode->io_tree, search_start,
 495                                      search_start + em_len - 1,
 496                                      EXTENT_DELALLOC_NEW,
 497                                      NULL, cached_state, GFP_NOFS);
 498 next:
 499                 search_start = extent_map_end(em);
 500                 free_extent_map(em);
 501                 if (ret)
 502                         return ret;
 503         }
 504         return 0;
 505 }
 506
 507 /*
 508  * after copy_from_user, pages need to be dirtied and we need to make
 509  * sure holes are created between the current EOF and the start of
 510  * any next extents (if required).
 511  *
 512  * this also makes the decision about creating an inline extent vs
 513  * doing real data extents, marking pages dirty and delalloc as required.
 514  */
 515 int btrfs_dirty_pages(struct inode *inode, struct page **pages,
 516                       size_t num_pages, loff_t pos, size_t write_bytes,
 517                       struct extent_state **cached)
 518 {
 519         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 520         int err = 0;
 521         int i;
 522         u64 num_bytes;
 523         u64 start_pos;
 524         u64 end_of_last_block;
 525         u64 end_pos = pos + write_bytes;
 526         loff_t isize = i_size_read(inode);
 527         unsigned int extra_bits = 0;
 528
 529         start_pos = pos & ~((u64) fs_info->sectorsize - 1);
 530         num_bytes = round_up(write_bytes + pos - start_pos,
 531                              fs_info->sectorsize);
 532
 533         end_of_last_block = start_pos + num_bytes - 1;
 534
 535         /*
 536          * The pages may have already been dirty, clear out old accounting so
 537          * we can set things up properly
 538          */
 539         clear_extent_bit(&BTRFS_I(inode)->io_tree, start_pos, end_of_last_block,
 540                          EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
 541                          0, 0, cached);
 542
 543         if (!btrfs_is_free_space_inode(BTRFS_I(inode))) {
 544                 if (start_pos >= isize &&
 545                     !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC)) {
 546                         /*
 547                          * There can't be any extents following eof in this case
 548                          * so just set the delalloc new bit for the range
 549                          * directly.
 550                          */
 551                         extra_bits |= EXTENT_DELALLOC_NEW;
 552                 } else {
 553                         err = btrfs_find_new_delalloc_bytes(BTRFS_I(inode),
 554                                                             start_pos,
 555                                                             num_bytes, cached);
 556                         if (err)
 557                                 return err;
 558                 }
 559         }
 560
 561         err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
 562                                         extra_bits, cached);
 563         if (err)
 564                 return err;
 565
 566         for (i = 0; i < num_pages; i++) {
 567                 struct page *p = pages[i];
 568                 SetPageUptodate(p);
 569                 ClearPageChecked(p);
 570                 set_page_dirty(p);
 571         }
 572
 573         /*
 574          * we've only changed i_size in ram, and we haven't updated
 575          * the disk i_size.  There is no need to log the inode
 576          * at this time.
 577          */
 578         if (end_pos > isize)
 579                 i_size_write(inode, end_pos);
 580         return 0;
 581 }
 582
 583 /*
 584  * this drops all the extents in the cache that intersect the range
 585  * [start, end].  Existing extents are split as required.
 586  */
 587 void btrfs_drop_extent_cache(struct btrfs_inode *inode, u64 start, u64 end,
 588                              int skip_pinned)
 589 {
 590         struct extent_map *em;
 591         struct extent_map *split = NULL;
 592         struct extent_map *split2 = NULL;
 593         struct extent_map_tree *em_tree = &inode->extent_tree;
 594         u64 len = end - start + 1;
 595         u64 gen;
 596         int ret;
 597         int testend = 1;
 598         unsigned long flags;
 599         int compressed = 0;
 600         bool modified;
 601
 602         WARN_ON(end < start);
 603         if (end == (u64)-1) {
 604                 len = (u64)-1;
 605                 testend = 0;
 606         }
 607         while (1) {
 608                 int no_splits = 0;
 609
 610                 modified = false;
 611                 if (!split)
 612                         split = alloc_extent_map();
 613                 if (!split2)
 614                         split2 = alloc_extent_map();
 615                 if (!split || !split2)
 616                         no_splits = 1;
 617
 618                 write_lock(&em_tree->lock);
 619                 em = lookup_extent_mapping(em_tree, start, len);
 620                 if (!em) {
 621                         write_unlock(&em_tree->lock);
 622                         break;
 623                 }
 624                 flags = em->flags;
 625                 gen = em->generation;
 626                 if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) {
 627                         if (testend && em->start + em->len >= start + len) {
 628                                 free_extent_map(em);
 629                                 write_unlock(&em_tree->lock);
 630                                 break;
 631                         }
 632                         start = em->start + em->len;
 633                         if (testend)
 634                                 len = start + len - (em->start + em->len);
 635                         free_extent_map(em);
 636                         write_unlock(&em_tree->lock);
 637                         continue;
 638                 }
 639                 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
 640                 clear_bit(EXTENT_FLAG_PINNED, &em->flags);
 641                 clear_bit(EXTENT_FLAG_LOGGING, &flags);
 642                 modified = !list_empty(&em->list);
 643                 if (no_splits)
 644                         goto next;
 645
 646                 if (em->start < start) {
 647                         split->start = em->start;
 648                         split->len = start - em->start;
 649
 650                         if (em->block_start < EXTENT_MAP_LAST_BYTE) {
 651                                 split->orig_start = em->orig_start;
 652                                 split->block_start = em->block_start;
 653
 654                                 if (compressed)
 655                                         split->block_len = em->block_len;
 656                                 else
 657                                         split->block_len = split->len;
 658                                 split->orig_block_len = max(split->block_len,
 659                                                 em->orig_block_len);
 660                                 split->ram_bytes = em->ram_bytes;
 661                         } else {
 662                                 split->orig_start = split->start;
 663                                 split->block_len = 0;
 664                                 split->block_start = em->block_start;
 665                                 split->orig_block_len = 0;
 666                                 split->ram_bytes = split->len;
 667                         }
 668
 669                         split->generation = gen;
 670                         split->bdev = em->bdev;
 671                         split->flags = flags;
 672                         split->compress_type = em->compress_type;
 673                         replace_extent_mapping(em_tree, em, split, modified);
 674                         free_extent_map(split);
 675                         split = split2;
 676                         split2 = NULL;
 677                 }
 678                 if (testend && em->start + em->len > start + len) {
 679                         u64 diff = start + len - em->start;
 680
 681                         split->start = start + len;
 682                         split->len = em->start + em->len - (start + len);
 683                         split->bdev = em->bdev;
 684                         split->flags = flags;
 685                         split->compress_type = em->compress_type;
 686                         split->generation = gen;
 687
 688                         if (em->block_start < EXTENT_MAP_LAST_BYTE) {
 689                                 split->orig_block_len = max(em->block_len,
 690                                                     em->orig_block_len);
 691
 692                                 split->ram_bytes = em->ram_bytes;
 693                                 if (compressed) {
 694                                         split->block_len = em->block_len;
 695                                         split->block_start = em->block_start;
 696                                         split->orig_start = em->orig_start;
 697                                 } else {
 698                                         split->block_len = split->len;
 699                                         split->block_start = em->block_start
 700                                                 + diff;
 701                                         split->orig_start = em->orig_start;
 702                                 }
 703                         } else {
 704                                 split->ram_bytes = split->len;
 705                                 split->orig_start = split->start;
 706                                 split->block_len = 0;
 707                                 split->block_start = em->block_start;
 708                                 split->orig_block_len = 0;
 709                         }
 710
 711                         if (extent_map_in_tree(em)) {
 712                                 replace_extent_mapping(em_tree, em, split,
 713                                                        modified);
 714                         } else {
 715                                 ret = add_extent_mapping(em_tree, split,
 716                                                          modified);
 717                                 ASSERT(ret == 0); /* Logic error */
 718                         }
 719                         free_extent_map(split);
 720                         split = NULL;
 721                 }
 722 next:
 723                 if (extent_map_in_tree(em))
 724                         remove_extent_mapping(em_tree, em);
 725                 write_unlock(&em_tree->lock);
 726
 727                 /* once for us */
 728                 free_extent_map(em);
 729                 /* once for the tree*/
 730                 free_extent_map(em);
 731         }
 732         if (split)
 733                 free_extent_map(split);
 734         if (split2)
 735                 free_extent_map(split2);
 736 }
 737
 738 /*
 739  * this is very complex, but the basic idea is to drop all extents
 740  * in the range start - end.  hint_block is filled in with a block number
 741  * that would be a good hint to the block allocator for this file.
 742  *
 743  * If an extent intersects the range but is not entirely inside the range
 744  * it is either truncated or split.  Anything entirely inside the range
 745  * is deleted from the tree.
 746  */
 747 int __btrfs_drop_extents(struct btrfs_trans_handle *trans,
 748                          struct btrfs_root *root, struct inode *inode,
 749                          struct btrfs_path *path, u64 start, u64 end,
 750                          u64 *drop_end, int drop_cache,
 751                          int replace_extent,
 752                          u32 extent_item_size,
 753                          int *key_inserted)
 754 {
 755         struct btrfs_fs_info *fs_info = root->fs_info;
 756         struct extent_buffer *leaf;
 757         struct btrfs_file_extent_item *fi;
 758         struct btrfs_ref ref = { 0 };
 759         struct btrfs_key key;
 760         struct btrfs_key new_key;
 761         u64 ino = btrfs_ino(BTRFS_I(inode));
 762         u64 search_start = start;
 763         u64 disk_bytenr = 0;
 764         u64 num_bytes = 0;
 765         u64 extent_offset = 0;
 766         u64 extent_end = 0;
 767         u64 last_end = start;
 768         int del_nr = 0;
 769         int del_slot = 0;
 770         int extent_type;
 771         int recow;
 772         int ret;
 773         int modify_tree = -1;
 774         int update_refs;
 775         int found = 0;
 776         int leafs_visited = 0;
 777
 778         if (drop_cache)
 779                 btrfs_drop_extent_cache(BTRFS_I(inode), start, end - 1, 0);
 780
 781         if (start >= BTRFS_I(inode)->disk_i_size && !replace_extent)
 782                 modify_tree = 0;
 783
 784         update_refs = (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
 785                        root == fs_info->tree_root);
 786         while (1) {
 787                 recow = 0;
 788                 ret = btrfs_lookup_file_extent(trans, root, path, ino,
 789                                                search_start, modify_tree);
 790                 if (ret < 0)
 791                         break;
 792                 if (ret > 0 && path->slots[0] > 0 && search_start == start) {
 793                         leaf = path->nodes[0];
 794                         btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
 795                         if (key.objectid == ino &&
 796                             key.type == BTRFS_EXTENT_DATA_KEY)
 797                                 path->slots[0]--;
 798                 }
 799                 ret = 0;
 800                 leafs_visited++;
 801 next_slot:
 802                 leaf = path->nodes[0];
 803                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
 804                         BUG_ON(del_nr > 0);
 805                         ret = btrfs_next_leaf(root, path);
 806                         if (ret < 0)
 807                                 break;
 808                         if (ret > 0) {
 809                                 ret = 0;
 810                                 break;
 811                         }
 812                         leafs_visited++;
 813                         leaf = path->nodes[0];
 814                         recow = 1;
 815                 }
 816
 817                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 818
 819                 if (key.objectid > ino)
 820                         break;
 821                 if (WARN_ON_ONCE(key.objectid < ino) ||
 822                     key.type < BTRFS_EXTENT_DATA_KEY) {
 823                         ASSERT(del_nr == 0);
 824                         path->slots[0]++;
 825                         goto next_slot;
 826                 }
 827                 if (key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= end)
 828                         break;
 829
 830                 fi = btrfs_item_ptr(leaf, path->slots[0],
 831                                     struct btrfs_file_extent_item);
 832                 extent_type = btrfs_file_extent_type(leaf, fi);
 833
 834                 if (extent_type == BTRFS_FILE_EXTENT_REG ||
 835                     extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
 836                         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
 837                         num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
 838                         extent_offset = btrfs_file_extent_offset(leaf, fi);
 839                         extent_end = key.offset +
 840                                 btrfs_file_extent_num_bytes(leaf, fi);
 841                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 842                         extent_end = key.offset +
 843                                 btrfs_file_extent_ram_bytes(leaf, fi);
 844                 } else {
 845                         /* can't happen */
 846                         BUG();
 847                 }
 848
 849                 /*
 850                  * Don't skip extent items representing 0 byte lengths. They
 851                  * used to be created (bug) if while punching holes we hit
 852                  * -ENOSPC condition. So if we find one here, just ensure we
 853                  * delete it, otherwise we would insert a new file extent item
 854                  * with the same key (offset) as that 0 bytes length file
 855                  * extent item in the call to setup_items_for_insert() later
 856                  * in this function.
 857                  */
 858                 if (extent_end == key.offset && extent_end >= search_start) {
 859                         last_end = extent_end;
 860                         goto delete_extent_item;
 861                 }
 862
 863                 if (extent_end <= search_start) {
 864                         path->slots[0]++;
 865                         goto next_slot;
 866                 }
 867
 868                 found = 1;
 869                 search_start = max(key.offset, start);
 870                 if (recow || !modify_tree) {
 871                         modify_tree = -1;
 872                         btrfs_release_path(path);
 873                         continue;
 874                 }
 875
 876                 /*
 877                  *     | - range to drop - |
 878                  *  | -------- extent -------- |
 879                  */
 880                 if (start > key.offset && end < extent_end) {
 881                         BUG_ON(del_nr > 0);
 882                         if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 883                                 ret = -EOPNOTSUPP;
 884                                 break;
 885                         }
 886
 887                         memcpy(&new_key, &key, sizeof(new_key));
 888                         new_key.offset = start;
 889                         ret = btrfs_duplicate_item(trans, root, path,
 890                                                    &new_key);
 891                         if (ret == -EAGAIN) {
 892                                 btrfs_release_path(path);
 893                                 continue;
 894                         }
 895                         if (ret < 0)
 896                                 break;
 897
 898                         leaf = path->nodes[0];
 899                         fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
 900                                             struct btrfs_file_extent_item);
 901                         btrfs_set_file_extent_num_bytes(leaf, fi,
 902                                                         start - key.offset);
 903
 904                         fi = btrfs_item_ptr(leaf, path->slots[0],
 905                                             struct btrfs_file_extent_item);
 906
 907                         extent_offset += start - key.offset;
 908                         btrfs_set_file_extent_offset(leaf, fi, extent_offset);
 909                         btrfs_set_file_extent_num_bytes(leaf, fi,
 910                                                         extent_end - start);
 911                         btrfs_mark_buffer_dirty(leaf);
 912
 913                         if (update_refs && disk_bytenr > 0) {
 914                                 btrfs_init_generic_ref(&ref,
 915                                                 BTRFS_ADD_DELAYED_REF,
 916                                                 disk_bytenr, num_bytes, 0);
 917                                 btrfs_init_data_ref(&ref,
 918                                                 root->root_key.objectid,
 919                                                 new_key.objectid,
 920                                                 start - extent_offset);
 921                                 ret = btrfs_inc_extent_ref(trans, &ref);
 922                                 BUG_ON(ret); /* -ENOMEM */
 923                         }
 924                         key.offset = start;
 925                 }
 926                 /*
 927                  * From here on out we will have actually dropped something, so
 928                  * last_end can be updated.
 929                  */
 930                 last_end = extent_end;
 931
 932                 /*
 933                  *  | ---- range to drop ----- |
 934                  *      | -------- extent -------- |
 935                  */
 936                 if (start <= key.offset && end < extent_end) {
 937                         if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 938                                 ret = -EOPNOTSUPP;
 939                                 break;
 940                         }
 941
 942                         memcpy(&new_key, &key, sizeof(new_key));
 943                         new_key.offset = end;
 944                         btrfs_set_item_key_safe(fs_info, path, &new_key);
 945
 946                         extent_offset += end - key.offset;
 947                         btrfs_set_file_extent_offset(leaf, fi, extent_offset);
 948                         btrfs_set_file_extent_num_bytes(leaf, fi,
 949                                                         extent_end - end);
 950                         btrfs_mark_buffer_dirty(leaf);
 951                         if (update_refs && disk_bytenr > 0)
 952                                 inode_sub_bytes(inode, end - key.offset);
 953                         break;
 954                 }
 955
 956                 search_start = extent_end;
 957                 /*
 958                  *       | ---- range to drop ----- |
 959                  *  | -------- extent -------- |
 960                  */
 961                 if (start > key.offset && end >= extent_end) {
 962                         BUG_ON(del_nr > 0);
 963                         if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 964                                 ret = -EOPNOTSUPP;
 965                                 break;
 966                         }
 967
 968                         btrfs_set_file_extent_num_bytes(leaf, fi,
 969                                                         start - key.offset);
 970                         btrfs_mark_buffer_dirty(leaf);
 971                         if (update_refs && disk_bytenr > 0)
 972                                 inode_sub_bytes(inode, extent_end - start);
 973                         if (end == extent_end)
 974                                 break;
 975
 976                         path->slots[0]++;
 977                         goto next_slot;
 978                 }
 979
 980                 /*
 981                  *  | ---- range to drop ----- |
 982                  *    | ------ extent ------ |
 983                  */
 984                 if (start <= key.offset && end >= extent_end) {
 985 delete_extent_item:
 986                         if (del_nr == 0) {
 987                                 del_slot = path->slots[0];
 988                                 del_nr = 1;
 989                         } else {
 990                                 BUG_ON(del_slot + del_nr != path->slots[0]);
 991                                 del_nr++;
 992                         }
 993
 994                         if (update_refs &&
 995                             extent_type == BTRFS_FILE_EXTENT_INLINE) {
 996                                 inode_sub_bytes(inode,
 997                                                 extent_end - key.offset);
 998                                 extent_end = ALIGN(extent_end,
 999                                                    fs_info->sectorsize);
1000                         } else if (update_refs && disk_bytenr > 0) {
1001                                 btrfs_init_generic_ref(&ref,
1002                                                 BTRFS_DROP_DELAYED_REF,
1003                                                 disk_bytenr, num_bytes, 0);
1004                                 btrfs_init_data_ref(&ref,
1005                                                 root->root_key.objectid,
1006                                                 key.objectid,
1007                                                 key.offset - extent_offset);
1008                                 ret = btrfs_free_extent(trans, &ref);
1009                                 BUG_ON(ret); /* -ENOMEM */
1010                                 inode_sub_bytes(inode,
1011                                                 extent_end - key.offset);
1012                         }
1013
1014                         if (end == extent_end)
1015                                 break;
1016
1017                         if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
1018                                 path->slots[0]++;
1019                                 goto next_slot;
1020                         }
1021
1022                         ret = btrfs_del_items(trans, root, path, del_slot,
1023                                               del_nr);
1024                         if (ret) {
1025                                 btrfs_abort_transaction(trans, ret);
1026                                 break;
1027                         }
1028
1029                         del_nr = 0;
1030                         del_slot = 0;
1031
1032                         btrfs_release_path(path);
1033                         continue;
1034                 }
1035
1036                 BUG();
1037         }
1038
1039         if (!ret && del_nr > 0) {
1040                 /*
1041                  * Set path->slots[0] to first slot, so that after the delete
1042                  * if items are move off from our leaf to its immediate left or
1043                  * right neighbor leafs, we end up with a correct and adjusted
1044                  * path->slots[0] for our insertion (if replace_extent != 0).
1045                  */
1046                 path->slots[0] = del_slot;
1047                 ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
1048                 if (ret)
1049                         btrfs_abort_transaction(trans, ret);
1050         }
1051
1052         leaf = path->nodes[0];
1053         /*
1054          * If btrfs_del_items() was called, it might have deleted a leaf, in
1055          * which case it unlocked our path, so check path->locks[0] matches a
1056          * write lock.
1057          */
1058         if (!ret && replace_extent && leafs_visited == 1 &&
1059             (path->locks[0] == BTRFS_WRITE_LOCK_BLOCKING ||
1060              path->locks[0] == BTRFS_WRITE_LOCK) &&
1061             btrfs_leaf_free_space(leaf) >=
1062             sizeof(struct btrfs_item) + extent_item_size) {
1063
1064                 key.objectid = ino;
1065                 key.type = BTRFS_EXTENT_DATA_KEY;
1066                 key.offset = start;
1067                 if (!del_nr && path->slots[0] < btrfs_header_nritems(leaf)) {
1068                         struct btrfs_key slot_key;
1069
1070                         btrfs_item_key_to_cpu(leaf, &slot_key, path->slots[0]);
1071                         if (btrfs_comp_cpu_keys(&key, &slot_key) > 0)
1072                                 path->slots[0]++;
1073                 }
1074                 setup_items_for_insert(root, path, &key,
1075                                        &extent_item_size,
1076                                        extent_item_size,
1077                                        sizeof(struct btrfs_item) +
1078                                        extent_item_size, 1);
1079                 *key_inserted = 1;
1080         }
1081
1082         if (!replace_extent || !(*key_inserted))
1083                 btrfs_release_path(path);
1084         if (drop_end)
1085                 *drop_end = found ? min(end, last_end) : end;
1086         return ret;
1087 }
1088
1089 int btrfs_drop_extents(struct btrfs_trans_handle *trans,
1090                        struct btrfs_root *root, struct inode *inode, u64 start,
1091                        u64 end, int drop_cache)
1092 {
1093         struct btrfs_path *path;
1094         int ret;
1095
1096         path = btrfs_alloc_path();
1097         if (!path)
1098                 return -ENOMEM;
1099         ret = __btrfs_drop_extents(trans, root, inode, path, start, end, NULL,
1100                                    drop_cache, 0, 0, NULL);
1101         btrfs_free_path(path);
1102         return ret;
1103 }
1104
1105 static int extent_mergeable(struct extent_buffer *leaf, int slot,
1106                             u64 objectid, u64 bytenr, u64 orig_offset,
1107                             u64 *start, u64 *end)
1108 {
1109         struct btrfs_file_extent_item *fi;
1110         struct btrfs_key key;
1111         u64 extent_end;
1112
1113         if (slot < 0 || slot >= btrfs_header_nritems(leaf))
1114                 return 0;
1115
1116         btrfs_item_key_to_cpu(leaf, &key, slot);
1117         if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
1118                 return 0;
1119
1120         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
1121         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
1122             btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
1123             btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
1124             btrfs_file_extent_compression(leaf, fi) ||
1125             btrfs_file_extent_encryption(leaf, fi) ||
1126             btrfs_file_extent_other_encoding(leaf, fi))
1127                 return 0;
1128
1129         extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1130         if ((*start && *start != key.offset) || (*end && *end != extent_end))
1131                 return 0;
1132
1133         *start = key.offset;
1134         *end = extent_end;
1135         return 1;
1136 }
1137
1138 /*
1139  * Mark extent in the range start - end as written.
1140  *
1141  * This changes extent type from 'pre-allocated' to 'regular'. If only
1142  * part of extent is marked as written, the extent will be split into
1143  * two or three.
1144  */
1145 int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
1146                               struct btrfs_inode *inode, u64 start, u64 end)
1147 {
1148         struct btrfs_fs_info *fs_info = trans->fs_info;
1149         struct btrfs_root *root = inode->root;
1150         struct extent_buffer *leaf;
1151         struct btrfs_path *path;
1152         struct btrfs_file_extent_item *fi;
1153         struct btrfs_ref ref = { 0 };
1154         struct btrfs_key key;
1155         struct btrfs_key new_key;
1156         u64 bytenr;
1157         u64 num_bytes;
1158         u64 extent_end;
1159         u64 orig_offset;
1160         u64 other_start;
1161         u64 other_end;
1162         u64 split;
1163         int del_nr = 0;
1164         int del_slot = 0;
1165         int recow;
1166         int ret = 0;
1167         u64 ino = btrfs_ino(inode);
1168
1169         path = btrfs_alloc_path();
1170         if (!path)
1171                 return -ENOMEM;
1172 again:
1173         recow = 0;
1174         split = start;
1175         key.objectid = ino;
1176         key.type = BTRFS_EXTENT_DATA_KEY;
1177         key.offset = split;
1178
1179         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1180         if (ret < 0)
1181                 goto out;
1182         if (ret > 0 && path->slots[0] > 0)
1183                 path->slots[0]--;
1184
1185         leaf = path->nodes[0];
1186         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1187         if (key.objectid != ino ||
1188             key.type != BTRFS_EXTENT_DATA_KEY) {
1189                 ret = -EINVAL;
1190                 btrfs_abort_transaction(trans, ret);
1191                 goto out;
1192         }
1193         fi = btrfs_item_ptr(leaf, path->slots[0],
1194                             struct btrfs_file_extent_item);
1195         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_PREALLOC) {
1196                 ret = -EINVAL;
1197                 btrfs_abort_transaction(trans, ret);
1198                 goto out;
1199         }
1200         extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1201         if (key.offset > start || extent_end < end) {
1202                 ret = -EINVAL;
1203                 btrfs_abort_transaction(trans, ret);
1204                 goto out;
1205         }
1206
1207         bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1208         num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
1209         orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
1210         memcpy(&new_key, &key, sizeof(new_key));
1211
1212         if (start == key.offset && end < extent_end) {
1213                 other_start = 0;
1214                 other_end = start;
1215                 if (extent_mergeable(leaf, path->slots[0] - 1,
1216                                      ino, bytenr, orig_offset,
1217                                      &other_start, &other_end)) {
1218                         new_key.offset = end;
1219                         btrfs_set_item_key_safe(fs_info, path, &new_key);
1220                         fi = btrfs_item_ptr(leaf, path->slots[0],
1221                                             struct btrfs_file_extent_item);
1222                         btrfs_set_file_extent_generation(leaf, fi,
1223                                                          trans->transid);
1224                         btrfs_set_file_extent_num_bytes(leaf, fi,
1225                                                         extent_end - end);
1226                         btrfs_set_file_extent_offset(leaf, fi,
1227                                                      end - orig_offset);
1228                         fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
1229                                             struct btrfs_file_extent_item);
1230                         btrfs_set_file_extent_generation(leaf, fi,
1231                                                          trans->transid);
1232                         btrfs_set_file_extent_num_bytes(leaf, fi,
1233                                                         end - other_start);
1234                         btrfs_mark_buffer_dirty(leaf);
1235                         goto out;
1236                 }
1237         }
1238
1239         if (start > key.offset && end == extent_end) {
1240                 other_start = end;
1241                 other_end = 0;
1242                 if (extent_mergeable(leaf, path->slots[0] + 1,
1243                                      ino, bytenr, orig_offset,
1244                                      &other_start, &other_end)) {
1245                         fi = btrfs_item_ptr(leaf, path->slots[0],
1246                                             struct btrfs_file_extent_item);
1247                         btrfs_set_file_extent_num_bytes(leaf, fi,
1248                                                         start - key.offset);
1249                         btrfs_set_file_extent_generation(leaf, fi,
1250                                                          trans->transid);
1251                         path->slots[0]++;
1252                         new_key.offset = start;
1253                         btrfs_set_item_key_safe(fs_info, path, &new_key);
1254
1255                         fi = btrfs_item_ptr(leaf, path->slots[0],
1256                                             struct btrfs_file_extent_item);
1257                         btrfs_set_file_extent_generation(leaf, fi,
1258                                                          trans->transid);
1259                         btrfs_set_file_extent_num_bytes(leaf, fi,
1260                                                         other_end - start);
1261                         btrfs_set_file_extent_offset(leaf, fi,
1262                                                      start - orig_offset);
1263                         btrfs_mark_buffer_dirty(leaf);
1264                         goto out;
1265                 }
1266         }
1267
1268         while (start > key.offset || end < extent_end) {
1269                 if (key.offset == start)
1270                         split = end;
1271
1272                 new_key.offset = split;
1273                 ret = btrfs_duplicate_item(trans, root, path, &new_key);
1274                 if (ret == -EAGAIN) {
1275                         btrfs_release_path(path);
1276                         goto again;
1277                 }
1278                 if (ret < 0) {
1279                         btrfs_abort_transaction(trans, ret);
1280                         goto out;
1281                 }
1282
1283                 leaf = path->nodes[0];
1284                 fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
1285                                     struct btrfs_file_extent_item);
1286                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1287                 btrfs_set_file_extent_num_bytes(leaf, fi,
1288                                                 split - key.offset);
1289
1290                 fi = btrfs_item_ptr(leaf, path->slots[0],
1291                                     struct btrfs_file_extent_item);
1292
1293                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1294                 btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
1295                 btrfs_set_file_extent_num_bytes(leaf, fi,
1296                                                 extent_end - split);
1297                 btrfs_mark_buffer_dirty(leaf);
1298
1299                 btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF, bytenr,
1300                                        num_bytes, 0);
1301                 btrfs_init_data_ref(&ref, root->root_key.objectid, ino,
1302                                     orig_offset);
1303                 ret = btrfs_inc_extent_ref(trans, &ref);
1304                 if (ret) {
1305                         btrfs_abort_transaction(trans, ret);
1306                         goto out;
1307                 }
1308
1309                 if (split == start) {
1310                         key.offset = start;
1311                 } else {
1312                         if (start != key.offset) {
1313                                 ret = -EINVAL;
1314                                 btrfs_abort_transaction(trans, ret);
1315                                 goto out;
1316                         }
1317                         path->slots[0]--;
1318                         extent_end = end;
1319                 }
1320                 recow = 1;
1321         }
1322
1323         other_start = end;
1324         other_end = 0;
1325         btrfs_init_generic_ref(&ref, BTRFS_DROP_DELAYED_REF, bytenr,
1326                                num_bytes, 0);
1327         btrfs_init_data_ref(&ref, root->root_key.objectid, ino, orig_offset);
1328         if (extent_mergeable(leaf, path->slots[0] + 1,
1329                              ino, bytenr, orig_offset,
1330                              &other_start, &other_end)) {
1331                 if (recow) {
1332                         btrfs_release_path(path);
1333                         goto again;
1334                 }
1335                 extent_end = other_end;
1336                 del_slot = path->slots[0] + 1;
1337                 del_nr++;
1338                 ret = btrfs_free_extent(trans, &ref);
1339                 if (ret) {
1340                         btrfs_abort_transaction(trans, ret);
1341                         goto out;
1342                 }
1343         }
1344         other_start = 0;
1345         other_end = start;
1346         if (extent_mergeable(leaf, path->slots[0] - 1,
1347                              ino, bytenr, orig_offset,
1348                              &other_start, &other_end)) {
1349                 if (recow) {
1350                         btrfs_release_path(path);
1351                         goto again;
1352                 }
1353                 key.offset = other_start;
1354                 del_slot = path->slots[0];
1355                 del_nr++;
1356                 ret = btrfs_free_extent(trans, &ref);
1357                 if (ret) {
1358                         btrfs_abort_transaction(trans, ret);
1359                         goto out;
1360                 }
1361         }
1362         if (del_nr == 0) {
1363                 fi = btrfs_item_ptr(leaf, path->slots[0],
1364                            struct btrfs_file_extent_item);
1365                 btrfs_set_file_extent_type(leaf, fi,
1366                                            BTRFS_FILE_EXTENT_REG);
1367                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1368                 btrfs_mark_buffer_dirty(leaf);
1369         } else {
1370                 fi = btrfs_item_ptr(leaf, del_slot - 1,
1371                            struct btrfs_file_extent_item);
1372                 btrfs_set_file_extent_type(leaf, fi,
1373                                            BTRFS_FILE_EXTENT_REG);
1374                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1375                 btrfs_set_file_extent_num_bytes(leaf, fi,
1376                                                 extent_end - key.offset);
1377                 btrfs_mark_buffer_dirty(leaf);
1378
1379                 ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
1380                 if (ret < 0) {
1381                         btrfs_abort_transaction(trans, ret);
1382                         goto out;
1383                 }
1384         }
1385 out:
1386         btrfs_free_path(path);
1387         return ret;
1388 }
1389
1390 /*
1391  * on error we return an unlocked page and the error value
1392  * on success we return a locked page and 0
1393  */
1394 static int prepare_uptodate_page(struct inode *inode,
1395                                  struct page *page, u64 pos,
1396                                  bool force_uptodate)
1397 {
1398         int ret = 0;
1399
1400         if (((pos & (PAGE_SIZE - 1)) || force_uptodate) &&
1401             !PageUptodate(page)) {
1402                 ret = btrfs_readpage(NULL, page);
1403                 if (ret)
1404                         return ret;
1405                 lock_page(page);
1406                 if (!PageUptodate(page)) {
1407                         unlock_page(page);
1408                         return -EIO;
1409                 }
1410                 if (page->mapping != inode->i_mapping) {
1411                         unlock_page(page);
1412                         return -EAGAIN;
1413                 }
1414         }
1415         return 0;
1416 }
1417
1418 /*
1419  * this just gets pages into the page cache and locks them down.
1420  */
1421 static noinline int prepare_pages(struct inode *inode, struct page **pages,
1422                                   size_t num_pages, loff_t pos,
1423                                   size_t write_bytes, bool force_uptodate)
1424 {
1425         int i;
1426         unsigned long index = pos >> PAGE_SHIFT;
1427         gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);
1428         int err = 0;
1429         int faili;
1430
1431         for (i = 0; i < num_pages; i++) {
1432 again:
1433                 pages[i] = find_or_create_page(inode->i_mapping, index + i,
1434                                                mask | __GFP_WRITE);
1435                 if (!pages[i]) {
1436                         faili = i - 1;
1437                         err = -ENOMEM;
1438                         goto fail;
1439                 }
1440
1441                 if (i == 0)
1442                         err = prepare_uptodate_page(inode, pages[i], pos,
1443                                                     force_uptodate);
1444                 if (!err && i == num_pages - 1)
1445                         err = prepare_uptodate_page(inode, pages[i],
1446                                                     pos + write_bytes, false);
1447                 if (err) {
1448                         put_page(pages[i]);
1449                         if (err == -EAGAIN) {
1450                                 err = 0;
1451                                 goto again;
1452                         }
1453                         faili = i - 1;
1454                         goto fail;
1455                 }
1456                 wait_on_page_writeback(pages[i]);
1457         }
1458
1459         return 0;
1460 fail:
1461         while (faili >= 0) {
1462                 unlock_page(pages[faili]);
1463                 put_page(pages[faili]);
1464                 faili--;
1465         }
1466         return err;
1467
1468 }
1469
1470 /*
1471  * This function locks the extent and properly waits for data=ordered extents
1472  * to finish before allowing the pages to be modified if need.
1473  *
1474  * The return value:
1475  * 1 - the extent is locked
1476  * 0 - the extent is not locked, and everything is OK
1477  * -EAGAIN - need re-prepare the pages
1478  * the other < 0 number - Something wrong happens
1479  */
1480 static noinline int
1481 lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages,
1482                                 size_t num_pages, loff_t pos,
1483                                 size_t write_bytes,
1484                                 u64 *lockstart, u64 *lockend,
1485                                 struct extent_state **cached_state)
1486 {
1487         struct btrfs_fs_info *fs_info = inode->root->fs_info;
1488         u64 start_pos;
1489         u64 last_pos;
1490         int i;
1491         int ret = 0;
1492
1493         start_pos = round_down(pos, fs_info->sectorsize);
1494         last_pos = start_pos
1495                 + round_up(pos + write_bytes - start_pos,
1496                            fs_info->sectorsize) - 1;
1497
1498         if (start_pos < inode->vfs_inode.i_size) {
1499                 struct btrfs_ordered_extent *ordered;
1500
1501                 lock_extent_bits(&inode->io_tree, start_pos, last_pos,
1502                                 cached_state);
1503                 ordered = btrfs_lookup_ordered_range(inode, start_pos,
1504                                                      last_pos - start_pos + 1);
1505                 if (ordered &&
1506                     ordered->file_offset + ordered->len > start_pos &&
1507                     ordered->file_offset <= last_pos) {
1508                         unlock_extent_cached(&inode->io_tree, start_pos,
1509                                         last_pos, cached_state);
1510                         for (i = 0; i < num_pages; i++) {
1511                                 unlock_page(pages[i]);
1512                                 put_page(pages[i]);
1513                         }
1514                         btrfs_start_ordered_extent(&inode->vfs_inode,
1515                                         ordered, 1);
1516                         btrfs_put_ordered_extent(ordered);
1517                         return -EAGAIN;
1518                 }
1519                 if (ordered)
1520                         btrfs_put_ordered_extent(ordered);
1521
1522                 *lockstart = start_pos;
1523                 *lockend = last_pos;
1524                 ret = 1;
1525         }
1526
1527         /*
1528          * It's possible the pages are dirty right now, but we don't want
1529          * to clean them yet because copy_from_user may catch a page fault
1530          * and we might have to fall back to one page at a time.  If that
1531          * happens, we'll unlock these pages and we'd have a window where
1532          * reclaim could sneak in and drop the once-dirty page on the floor
1533          * without writing it.
1534          *
1535          * We have the pages locked and the extent range locked, so there's
1536          * no way someone can start IO on any dirty pages in this range.
1537          *
1538          * We'll call btrfs_dirty_pages() later on, and that will flip around
1539          * delalloc bits and dirty the pages as required.
1540          */
1541         for (i = 0; i < num_pages; i++) {
1542                 set_page_extent_mapped(pages[i]);
1543                 WARN_ON(!PageLocked(pages[i]));
1544         }
1545
1546         return ret;
1547 }
1548
1549 int btrfs_check_can_nocow(struct btrfs_inode *inode, loff_t pos,
1550                           size_t *write_bytes)
1551 {
1552         struct btrfs_fs_info *fs_info = inode->root->fs_info;
1553         struct btrfs_root *root = inode->root;
1554         u64 lockstart, lockend;
1555         u64 num_bytes;
1556         int ret;
1557
1558         ret = btrfs_start_write_no_snapshotting(root);
1559         if (!ret)
1560                 return -EAGAIN;
1561
1562         lockstart = round_down(pos, fs_info->sectorsize);
1563         lockend = round_up(pos + *write_bytes,
1564                            fs_info->sectorsize) - 1;
1565
1566         btrfs_lock_and_flush_ordered_range(&inode->io_tree, inode, lockstart,
1567                                            lockend, NULL);
1568
1569         num_bytes = lockend - lockstart + 1;
1570         ret = can_nocow_extent(&inode->vfs_inode, lockstart, &num_bytes,
1571                         NULL, NULL, NULL, false);
1572         if (ret <= 0) {
1573                 ret = 0;
1574                 btrfs_end_write_no_snapshotting(root);
1575         } else {
1576                 *write_bytes = min_t(size_t, *write_bytes ,
1577                                      num_bytes - pos + lockstart);
1578         }
1579
1580         unlock_extent(&inode->io_tree, lockstart, lockend);
1581
1582         return ret;
1583 }
1584
1585 static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb,
1586                                                struct iov_iter *i)
1587 {
1588         struct file *file = iocb->ki_filp;
1589         loff_t pos = iocb->ki_pos;
1590         struct inode *inode = file_inode(file);
1591         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1592         struct btrfs_root *root = BTRFS_I(inode)->root;
1593         struct page **pages = NULL;
1594         struct extent_changeset *data_reserved = NULL;
1595         u64 release_bytes = 0;
1596         u64 lockstart;
1597         u64 lockend;
1598         size_t num_written = 0;
1599         int nrptrs;
1600         int ret = 0;
1601         bool only_release_metadata = false;
1602         bool force_page_uptodate = false;
1603
1604         nrptrs = min(DIV_ROUND_UP(iov_iter_count(i), PAGE_SIZE),
1605                         PAGE_SIZE / (sizeof(struct page *)));
1606         nrptrs = min(nrptrs, current->nr_dirtied_pause - current->nr_dirtied);
1607         nrptrs = max(nrptrs, 8);
1608         pages = kmalloc_array(nrptrs, sizeof(struct page *), GFP_KERNEL);
1609         if (!pages)
1610                 return -ENOMEM;
1611
1612         while (iov_iter_count(i) > 0) {
1613                 struct extent_state *cached_state = NULL;
1614                 size_t offset = offset_in_page(pos);
1615                 size_t sector_offset;
1616                 size_t write_bytes = min(iov_iter_count(i),
1617                                          nrptrs * (size_t)PAGE_SIZE -
1618                                          offset);
1619                 size_t num_pages = DIV_ROUND_UP(write_bytes + offset,
1620                                                 PAGE_SIZE);
1621                 size_t reserve_bytes;
1622                 size_t dirty_pages;
1623                 size_t copied;
1624                 size_t dirty_sectors;
1625                 size_t num_sectors;
1626                 int extents_locked;
1627
1628                 WARN_ON(num_pages > nrptrs);
1629
1630                 /*
1631                  * Fault pages before locking them in prepare_pages
1632                  * to avoid recursive lock
1633                  */
1634                 if (unlikely(iov_iter_fault_in_readable(i, write_bytes))) {
1635                         ret = -EFAULT;
1636                         break;
1637                 }
1638
1639                 only_release_metadata = false;
1640                 sector_offset = pos & (fs_info->sectorsize - 1);
1641                 reserve_bytes = round_up(write_bytes + sector_offset,
1642                                 fs_info->sectorsize);
1643
1644                 extent_changeset_release(data_reserved);
1645                 ret = btrfs_check_data_free_space(inode, &data_reserved, pos,
1646                                                   write_bytes);
1647                 if (ret < 0) {
1648                         if ((BTRFS_I(inode)->flags & (BTRFS_INODE_NODATACOW |
1649                                                       BTRFS_INODE_PREALLOC)) &&
1650                             btrfs_check_can_nocow(BTRFS_I(inode), pos,
1651                                         &write_bytes) > 0) {
1652                                 /*
1653                                  * For nodata cow case, no need to reserve
1654                                  * data space.
1655                                  */
1656                                 only_release_metadata = true;
1657                                 /*
1658                                  * our prealloc extent may be smaller than
1659                                  * write_bytes, so scale down.
1660                                  */
1661                                 num_pages = DIV_ROUND_UP(write_bytes + offset,
1662                                                          PAGE_SIZE);
1663                                 reserve_bytes = round_up(write_bytes +
1664                                                          sector_offset,
1665                                                          fs_info->sectorsize);
1666                         } else {
1667                                 break;
1668                         }
1669                 }
1670
1671                 WARN_ON(reserve_bytes == 0);
1672                 ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode),
1673                                 reserve_bytes);
1674                 if (ret) {
1675                         if (!only_release_metadata)
1676                                 btrfs_free_reserved_data_space(inode,
1677                                                 data_reserved, pos,
1678                                                 write_bytes);
1679                         else
1680                                 btrfs_end_write_no_snapshotting(root);
1681                         break;
1682                 }
1683
1684                 release_bytes = reserve_bytes;
1685 again:
1686                 /*
1687                  * This is going to setup the pages array with the number of
1688                  * pages we want, so we don't really need to worry about the
1689                  * contents of pages from loop to loop
1690                  */
1691                 ret = prepare_pages(inode, pages, num_pages,
1692                                     pos, write_bytes,
1693                                     force_page_uptodate);
1694                 if (ret) {
1695                         btrfs_delalloc_release_extents(BTRFS_I(inode),
1696                                                        reserve_bytes);
1697                         break;
1698                 }
1699
1700                 extents_locked = lock_and_cleanup_extent_if_need(
1701                                 BTRFS_I(inode), pages,
1702                                 num_pages, pos, write_bytes, &lockstart,
1703                                 &lockend, &cached_state);
1704                 if (extents_locked < 0) {
1705                         if (extents_locked == -EAGAIN)
1706                                 goto again;
1707                         btrfs_delalloc_release_extents(BTRFS_I(inode),
1708                                                        reserve_bytes);
1709                         ret = extents_locked;
1710                         break;
1711                 }
1712
1713                 copied = btrfs_copy_from_user(pos, write_bytes, pages, i);
1714
1715                 num_sectors = BTRFS_BYTES_TO_BLKS(fs_info, reserve_bytes);
1716                 dirty_sectors = round_up(copied + sector_offset,
1717                                         fs_info->sectorsize);
1718                 dirty_sectors = BTRFS_BYTES_TO_BLKS(fs_info, dirty_sectors);
1719
1720                 /*
1721                  * if we have trouble faulting in the pages, fall
1722                  * back to one page at a time
1723                  */
1724                 if (copied < write_bytes)
1725                         nrptrs = 1;
1726
1727                 if (copied == 0) {
1728                         force_page_uptodate = true;
1729                         dirty_sectors = 0;
1730                         dirty_pages = 0;
1731                 } else {
1732                         force_page_uptodate = false;
1733                         dirty_pages = DIV_ROUND_UP(copied + offset,
1734                                                    PAGE_SIZE);
1735                 }
1736
1737                 if (num_sectors > dirty_sectors) {
1738                         /* release everything except the sectors we dirtied */
1739                         release_bytes -= dirty_sectors <<
1740                                                 fs_info->sb->s_blocksize_bits;
1741                         if (only_release_metadata) {
1742                                 btrfs_delalloc_release_metadata(BTRFS_I(inode),
1743                                                         release_bytes, true);
1744                         } else {
1745                                 u64 __pos;
1746
1747                                 __pos = round_down(pos,
1748                                                    fs_info->sectorsize) +
1749                                         (dirty_pages << PAGE_SHIFT);
1750                                 btrfs_delalloc_release_space(inode,
1751                                                 data_reserved, __pos,
1752                                                 release_bytes, true);
1753                         }
1754                 }
1755
1756                 release_bytes = round_up(copied + sector_offset,
1757                                         fs_info->sectorsize);
1758
1759                 if (copied > 0)
1760                         ret = btrfs_dirty_pages(inode, pages, dirty_pages,
1761                                                 pos, copied, &cached_state);
1762
1763                 /*
1764                  * If we have not locked the extent range, because the range's
1765                  * start offset is >= i_size, we might still have a non-NULL
1766                  * cached extent state, acquired while marking the extent range
1767                  * as delalloc through btrfs_dirty_pages(). Therefore free any
1768                  * possible cached extent state to avoid a memory leak.
1769                  */
1770                 if (extents_locked)
1771                         unlock_extent_cached(&BTRFS_I(inode)->io_tree,
1772                                              lockstart, lockend, &cached_state);
1773                 else
1774                         free_extent_state(cached_state);
1775
1776                 btrfs_delalloc_release_extents(BTRFS_I(inode), reserve_bytes);
1777                 if (ret) {
1778                         btrfs_drop_pages(pages, num_pages);
1779                         break;
1780                 }
1781
1782                 release_bytes = 0;
1783                 if (only_release_metadata)
1784                         btrfs_end_write_no_snapshotting(root);
1785
1786                 if (only_release_metadata && copied > 0) {
1787                         lockstart = round_down(pos,
1788                                                fs_info->sectorsize);
1789                         lockend = round_up(pos + copied,
1790                                            fs_info->sectorsize) - 1;
1791
1792                         set_extent_bit(&BTRFS_I(inode)->io_tree, lockstart,
1793                                        lockend, EXTENT_NORESERVE, NULL,
1794                                        NULL, GFP_NOFS);
1795                 }
1796
1797                 btrfs_drop_pages(pages, num_pages);
1798
1799                 cond_resched();
1800
1801                 balance_dirty_pages_ratelimited(inode->i_mapping);
1802                 if (dirty_pages < (fs_info->nodesize >> PAGE_SHIFT) + 1)
1803                         btrfs_btree_balance_dirty(fs_info);
1804
1805                 pos += copied;
1806                 num_written += copied;
1807         }
1808
1809         kfree(pages);
1810
1811         if (release_bytes) {
1812                 if (only_release_metadata) {
1813                         btrfs_end_write_no_snapshotting(root);
1814                         btrfs_delalloc_release_metadata(BTRFS_I(inode),
1815                                         release_bytes, true);
1816                 } else {
1817                         btrfs_delalloc_release_space(inode, data_reserved,
1818                                         round_down(pos, fs_info->sectorsize),
1819                                         release_bytes, true);
1820                 }
1821         }
1822
1823         extent_changeset_free(data_reserved);
1824         return num_written ? num_written : ret;
1825 }
1826
1827 static ssize_t __btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from)
1828 {
1829         struct file *file = iocb->ki_filp;
1830         struct inode *inode = file_inode(file);
1831         loff_t pos;
1832         ssize_t written;
1833         ssize_t written_buffered;
1834         loff_t endbyte;
1835         int err;
1836
1837         written = generic_file_direct_write(iocb, from);
1838
1839         if (written < 0 || !iov_iter_count(from))
1840                 return written;
1841
1842         pos = iocb->ki_pos;
1843         written_buffered = btrfs_buffered_write(iocb, from);
1844         if (written_buffered < 0) {
1845                 err = written_buffered;
1846                 goto out;
1847         }
1848         /*
1849          * Ensure all data is persisted. We want the next direct IO read to be
1850          * able to read what was just written.
1851          */
1852         endbyte = pos + written_buffered - 1;
1853         err = btrfs_fdatawrite_range(inode, pos, endbyte);
1854         if (err)
1855                 goto out;
1856         err = filemap_fdatawait_range(inode->i_mapping, pos, endbyte);
1857         if (err)
1858                 goto out;
1859         written += written_buffered;
1860         iocb->ki_pos = pos + written_buffered;
1861         invalidate_mapping_pages(file->f_mapping, pos >> PAGE_SHIFT,
1862                                  endbyte >> PAGE_SHIFT);
1863 out:
1864         return written ? written : err;
1865 }
1866
1867 static void update_time_for_write(struct inode *inode)
1868 {
1869         struct timespec64 now;
1870
1871         if (IS_NOCMTIME(inode))
1872                 return;
1873
1874         now = current_time(inode);
1875         if (!timespec64_equal(&inode->i_mtime, &now))
1876                 inode->i_mtime = now;
1877
1878         if (!timespec64_equal(&inode->i_ctime, &now))
1879                 inode->i_ctime = now;
1880
1881         if (IS_I_VERSION(inode))
1882                 inode_inc_iversion(inode);
1883 }
1884
1885 static ssize_t btrfs_file_write_iter(struct kiocb *iocb,
1886                                     struct iov_iter *from)
1887 {
1888         struct file *file = iocb->ki_filp;
1889         struct inode *inode = file_inode(file);
1890         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1891         struct btrfs_root *root = BTRFS_I(inode)->root;
1892         u64 start_pos;
1893         u64 end_pos;
1894         ssize_t num_written = 0;
1895         const bool sync = iocb->ki_flags & IOCB_DSYNC;
1896         ssize_t err;
1897         loff_t pos;
1898         size_t count;
1899         loff_t oldsize;
1900         int clean_page = 0;
1901
1902         if (!(iocb->ki_flags & IOCB_DIRECT) &&
1903             (iocb->ki_flags & IOCB_NOWAIT))
1904                 return -EOPNOTSUPP;
1905
1906         if (iocb->ki_flags & IOCB_NOWAIT) {
1907                 if (!inode_trylock(inode))
1908                         return -EAGAIN;
1909         } else {
1910                 inode_lock(inode);
1911         }
1912
1913         err = generic_write_checks(iocb, from);
1914         if (err <= 0) {
1915                 inode_unlock(inode);
1916                 return err;
1917         }
1918
1919         pos = iocb->ki_pos;
1920         count = iov_iter_count(from);
1921         if (iocb->ki_flags & IOCB_NOWAIT) {
1922                 size_t nocow_bytes = count;
1923
1924                 /*
1925                  * We will allocate space in case nodatacow is not set,
1926                  * so bail
1927                  */
1928                 if (!(BTRFS_I(inode)->flags & (BTRFS_INODE_NODATACOW |
1929                                               BTRFS_INODE_PREALLOC)) ||
1930                     btrfs_check_can_nocow(BTRFS_I(inode), pos,
1931                                           &nocow_bytes) <= 0) {
1932                         inode_unlock(inode);
1933                         return -EAGAIN;
1934                 }
1935
1936                 /* check_can_nocow() locks the snapshot lock on success */
1937                 btrfs_end_write_no_snapshotting(root);
1938                 /*
1939                  * There are holes in the range or parts of the range that must
1940                  * be COWed (shared extents, RO block groups, etc), so just bail
1941                  * out.
1942                  */
1943                 if (nocow_bytes < count) {
1944                         inode_unlock(inode);
1945                         return -EAGAIN;
1946                 }
1947         }
1948
1949         current->backing_dev_info = inode_to_bdi(inode);
1950         err = file_remove_privs(file);
1951         if (err) {
1952                 inode_unlock(inode);
1953                 goto out;
1954         }
1955
1956         /*
1957          * If BTRFS flips readonly due to some impossible error
1958          * (fs_info->fs_state now has BTRFS_SUPER_FLAG_ERROR),
1959          * although we have opened a file as writable, we have
1960          * to stop this write operation to ensure FS consistency.
1961          */
1962         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) {
1963                 inode_unlock(inode);
1964                 err = -EROFS;
1965                 goto out;
1966         }
1967
1968         /*
1969          * We reserve space for updating the inode when we reserve space for the
1970          * extent we are going to write, so we will enospc out there.  We don't
1971          * need to start yet another transaction to update the inode as we will
1972          * update the inode when we finish writing whatever data we write.
1973          */
1974         update_time_for_write(inode);
1975
1976         start_pos = round_down(pos, fs_info->sectorsize);
1977         oldsize = i_size_read(inode);
1978         if (start_pos > oldsize) {
1979                 /* Expand hole size to cover write data, preventing empty gap */
1980                 end_pos = round_up(pos + count,
1981                                    fs_info->sectorsize);
1982                 err = btrfs_cont_expand(inode, oldsize, end_pos);
1983                 if (err) {
1984                         inode_unlock(inode);
1985                         goto out;
1986                 }
1987                 if (start_pos > round_up(oldsize, fs_info->sectorsize))
1988                         clean_page = 1;
1989         }
1990
1991         if (sync)
1992                 atomic_inc(&BTRFS_I(inode)->sync_writers);
1993
1994         if (iocb->ki_flags & IOCB_DIRECT) {
1995                 num_written = __btrfs_direct_write(iocb, from);
1996         } else {
1997                 num_written = btrfs_buffered_write(iocb, from);
1998                 if (num_written > 0)
1999                         iocb->ki_pos = pos + num_written;
2000                 if (clean_page)
2001                         pagecache_isize_extended(inode, oldsize,
2002                                                 i_size_read(inode));
2003         }
2004
2005         inode_unlock(inode);
2006
2007         btrfs_set_inode_last_sub_trans(BTRFS_I(inode));
2008
2009         if (num_written > 0)
2010                 num_written = generic_write_sync(iocb, num_written);
2011
2012         if (sync)
2013                 atomic_dec(&BTRFS_I(inode)->sync_writers);
2014 out:
2015         current->backing_dev_info = NULL;
2016         return num_written ? num_written : err;
2017 }
2018
2019 int btrfs_release_file(struct inode *inode, struct file *filp)
2020 {
2021         struct btrfs_file_private *private = filp->private_data;
2022
2023         if (private && private->filldir_buf)
2024                 kfree(private->filldir_buf);
2025         kfree(private);
2026         filp->private_data = NULL;
2027
2028         /*
2029          * ordered_data_close is set by setattr when we are about to truncate
2030          * a file from a non-zero size to a zero size.  This tries to
2031          * flush down new bytes that may have been written if the
2032          * application were using truncate to replace a file in place.
2033          */
2034         if (test_and_clear_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
2035                                &BTRFS_I(inode)->runtime_flags))
2036                         filemap_flush(inode->i_mapping);
2037         return 0;
2038 }
2039
2040 static int start_ordered_ops(struct inode *inode, loff_t start, loff_t end)
2041 {
2042         int ret;
2043         struct blk_plug plug;
2044
2045         /*
2046          * This is only called in fsync, which would do synchronous writes, so
2047          * a plug can merge adjacent IOs as much as possible.  Esp. in case of
2048          * multiple disks using raid profile, a large IO can be split to
2049          * several segments of stripe length (currently 64K).
2050          */
2051         blk_start_plug(&plug);
2052         atomic_inc(&BTRFS_I(inode)->sync_writers);
2053         ret = btrfs_fdatawrite_range(inode, start, end);
2054         atomic_dec(&BTRFS_I(inode)->sync_writers);
2055         blk_finish_plug(&plug);
2056
2057         return ret;
2058 }
2059
2060 /*
2061  * fsync call for both files and directories.  This logs the inode into
2062  * the tree log instead of forcing full commits whenever possible.
2063  *
2064  * It needs to call filemap_fdatawait so that all ordered extent updates are
2065  * in the metadata btree are up to date for copying to the log.
2066  *
2067  * It drops the inode mutex before doing the tree log commit.  This is an
2068  * important optimization for directories because holding the mutex prevents
2069  * new operations on the dir while we write to disk.
2070  */
2071 int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
2072 {
2073         struct dentry *dentry = file_dentry(file);
2074         struct inode *inode = d_inode(dentry);
2075         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2076         struct btrfs_root *root = BTRFS_I(inode)->root;
2077         struct btrfs_trans_handle *trans;
2078         struct btrfs_log_ctx ctx;
2079         int ret = 0, err;
2080
2081         trace_btrfs_sync_file(file, datasync);
2082
2083         btrfs_init_log_ctx(&ctx, inode);
2084
2085         /*
2086          * Set the range to full if the NO_HOLES feature is not enabled.
2087          * This is to avoid missing file extent items representing holes after
2088          * replaying the log.
2089          */
2090         if (!btrfs_fs_incompat(fs_info, NO_HOLES)) {
2091                 start = 0;
2092                 end = LLONG_MAX;
2093         }
2094
2095         /*
2096          * We write the dirty pages in the range and wait until they complete
2097          * out of the ->i_mutex. If so, we can flush the dirty pages by
2098          * multi-task, and make the performance up.  See
2099          * btrfs_wait_ordered_range for an explanation of the ASYNC check.
2100          */
2101         ret = start_ordered_ops(inode, start, end);
2102         if (ret)
2103                 goto out;
2104
2105         inode_lock(inode);
2106
2107         /*
2108          * We take the dio_sem here because the tree log stuff can race with
2109          * lockless dio writes and get an extent map logged for an extent we
2110          * never waited on.  We need it this high up for lockdep reasons.
2111          */
2112         down_write(&BTRFS_I(inode)->dio_sem);
2113
2114         atomic_inc(&root->log_batch);
2115
2116         /*
2117          * If the inode needs a full sync, make sure we use a full range to
2118          * avoid log tree corruption, due to hole detection racing with ordered
2119          * extent completion for adjacent ranges, and assertion failures during
2120          * hole detection. Do this while holding the inode lock, to avoid races
2121          * with other tasks.
2122          */
2123         if (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2124                      &BTRFS_I(inode)->runtime_flags)) {
2125                 start = 0;
2126                 end = LLONG_MAX;
2127         }
2128
2129         /*
2130          * Before we acquired the inode's lock, someone may have dirtied more
2131          * pages in the target range. We need to make sure that writeback for
2132          * any such pages does not start while we are logging the inode, because
2133          * if it does, any of the following might happen when we are not doing a
2134          * full inode sync:
2135          *
2136          * 1) We log an extent after its writeback finishes but before its
2137          *    checksums are added to the csum tree, leading to -EIO errors
2138          *    when attempting to read the extent after a log replay.
2139          *
2140          * 2) We can end up logging an extent before its writeback finishes.
2141          *    Therefore after the log replay we will have a file extent item
2142          *    pointing to an unwritten extent (and no data checksums as well).
2143          *
2144          * So trigger writeback for any eventual new dirty pages and then we
2145          * wait for all ordered extents to complete below.
2146          */
2147         ret = start_ordered_ops(inode, start, end);
2148         if (ret) {
2149                 up_write(&BTRFS_I(inode)->dio_sem);
2150                 inode_unlock(inode);
2151                 goto out;
2152         }
2153
2154         /*
2155          * We have to do this here to avoid the priority inversion of waiting on
2156          * IO of a lower priority task while holding a transaction open.
2157          *
2158          * Also, the range length can be represented by u64, we have to do the
2159          * typecasts to avoid signed overflow if it's [0, LLONG_MAX].
2160          */
2161         ret = btrfs_wait_ordered_range(inode, start, (u64)end - (u64)start + 1);
2162         if (ret) {
2163                 up_write(&BTRFS_I(inode)->dio_sem);
2164                 inode_unlock(inode);
2165                 goto out;
2166         }
2167         atomic_inc(&root->log_batch);
2168
2169         smp_mb();
2170         if (btrfs_inode_in_log(BTRFS_I(inode), fs_info->generation) ||
2171             BTRFS_I(inode)->last_trans <= fs_info->last_trans_committed) {
2172                 /*
2173                  * We've had everything committed since the last time we were
2174                  * modified so clear this flag in case it was set for whatever
2175                  * reason, it's no longer relevant.
2176                  */
2177                 clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2178                           &BTRFS_I(inode)->runtime_flags);
2179                 /*
2180                  * An ordered extent might have started before and completed
2181                  * already with io errors, in which case the inode was not
2182                  * updated and we end up here. So check the inode's mapping
2183                  * for any errors that might have happened since we last
2184                  * checked called fsync.
2185                  */
2186                 ret = filemap_check_wb_err(inode->i_mapping, file->f_wb_err);
2187                 up_write(&BTRFS_I(inode)->dio_sem);
2188                 inode_unlock(inode);
2189                 goto out;
2190         }
2191
2192         /*
2193          * We use start here because we will need to wait on the IO to complete
2194          * in btrfs_sync_log, which could require joining a transaction (for
2195          * example checking cross references in the nocow path).  If we use join
2196          * here we could get into a situation where we're waiting on IO to
2197          * happen that is blocked on a transaction trying to commit.  With start
2198          * we inc the extwriter counter, so we wait for all extwriters to exit
2199          * before we start blocking joiners.  This comment is to keep somebody
2200          * from thinking they are super smart and changing this to
2201          * btrfs_join_transaction *cough*Josef*cough*.
2202          */
2203         trans = btrfs_start_transaction(root, 0);
2204         if (IS_ERR(trans)) {
2205                 ret = PTR_ERR(trans);
2206                 up_write(&BTRFS_I(inode)->dio_sem);
2207                 inode_unlock(inode);
2208                 goto out;
2209         }
2210
2211         ret = btrfs_log_dentry_safe(trans, dentry, start, end, &ctx);
2212         if (ret < 0) {
2213                 /* Fallthrough and commit/free transaction. */
2214                 ret = 1;
2215         }
2216
2217         /* we've logged all the items and now have a consistent
2218          * version of the file in the log.  It is possible that
2219          * someone will come in and modify the file, but that's
2220          * fine because the log is consistent on disk, and we
2221          * have references to all of the file's extents
2222          *
2223          * It is possible that someone will come in and log the
2224          * file again, but that will end up using the synchronization
2225          * inside btrfs_sync_log to keep things safe.
2226          */
2227         up_write(&BTRFS_I(inode)->dio_sem);
2228         inode_unlock(inode);
2229
2230         if (ret != BTRFS_NO_LOG_SYNC) {
2231                 if (!ret) {
2232                         ret = btrfs_sync_log(trans, root, &ctx);
2233                         if (!ret) {
2234                                 ret = btrfs_end_transaction(trans);
2235                                 goto out;
2236                         }
2237                 }
2238                 ret = btrfs_commit_transaction(trans);
2239         } else {
2240                 ret = btrfs_end_transaction(trans);
2241         }
2242 out:
2243         ASSERT(list_empty(&ctx.list));
2244         err = file_check_and_advance_wb_err(file);
2245         if (!ret)
2246                 ret = err;
2247         return ret > 0 ? -EIO : ret;
2248 }
2249
2250 static const struct vm_operations_struct btrfs_file_vm_ops = {
2251         .fault          = filemap_fault,
2252         .map_pages      = filemap_map_pages,
2253         .page_mkwrite   = btrfs_page_mkwrite,
2254 };
2255
2256 static int btrfs_file_mmap(struct file  *filp, struct vm_area_struct *vma)
2257 {
2258         struct address_space *mapping = filp->f_mapping;
2259
2260         if (!mapping->a_ops->readpage)
2261                 return -ENOEXEC;
2262
2263         file_accessed(filp);
2264         vma->vm_ops = &btrfs_file_vm_ops;
2265
2266         return 0;
2267 }
2268
2269 static int hole_mergeable(struct btrfs_inode *inode, struct extent_buffer *leaf,
2270                           int slot, u64 start, u64 end)
2271 {
2272         struct btrfs_file_extent_item *fi;
2273         struct btrfs_key key;
2274
2275         if (slot < 0 || slot >= btrfs_header_nritems(leaf))
2276                 return 0;
2277
2278         btrfs_item_key_to_cpu(leaf, &key, slot);
2279         if (key.objectid != btrfs_ino(inode) ||
2280             key.type != BTRFS_EXTENT_DATA_KEY)
2281                 return 0;
2282
2283         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2284
2285         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2286                 return 0;
2287
2288         if (btrfs_file_extent_disk_bytenr(leaf, fi))
2289                 return 0;
2290
2291         if (key.offset == end)
2292                 return 1;
2293         if (key.offset + btrfs_file_extent_num_bytes(leaf, fi) == start)
2294                 return 1;
2295         return 0;
2296 }
2297
2298 static int fill_holes(struct btrfs_trans_handle *trans,
2299                 struct btrfs_inode *inode,
2300                 struct btrfs_path *path, u64 offset, u64 end)
2301 {
2302         struct btrfs_fs_info *fs_info = trans->fs_info;
2303         struct btrfs_root *root = inode->root;
2304         struct extent_buffer *leaf;
2305         struct btrfs_file_extent_item *fi;
2306         struct extent_map *hole_em;
2307         struct extent_map_tree *em_tree = &inode->extent_tree;
2308         struct btrfs_key key;
2309         int ret;
2310
2311         if (btrfs_fs_incompat(fs_info, NO_HOLES))
2312                 goto out;
2313
2314         key.objectid = btrfs_ino(inode);
2315         key.type = BTRFS_EXTENT_DATA_KEY;
2316         key.offset = offset;
2317
2318         ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2319         if (ret <= 0) {
2320                 /*
2321                  * We should have dropped this offset, so if we find it then
2322                  * something has gone horribly wrong.
2323                  */
2324                 if (ret == 0)
2325                         ret = -EINVAL;
2326                 return ret;
2327         }
2328
2329         leaf = path->nodes[0];
2330         if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) {
2331                 u64 num_bytes;
2332
2333                 path->slots[0]--;
2334                 fi = btrfs_item_ptr(leaf, path->slots[0],
2335                                     struct btrfs_file_extent_item);
2336                 num_bytes = btrfs_file_extent_num_bytes(leaf, fi) +
2337                         end - offset;
2338                 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2339                 btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2340                 btrfs_set_file_extent_offset(leaf, fi, 0);
2341                 btrfs_mark_buffer_dirty(leaf);
2342                 goto out;
2343         }
2344
2345         if (hole_mergeable(inode, leaf, path->slots[0], offset, end)) {
2346                 u64 num_bytes;
2347
2348                 key.offset = offset;
2349                 btrfs_set_item_key_safe(fs_info, path, &key);
2350                 fi = btrfs_item_ptr(leaf, path->slots[0],
2351                                     struct btrfs_file_extent_item);
2352                 num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end -
2353                         offset;
2354                 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2355                 btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2356                 btrfs_set_file_extent_offset(leaf, fi, 0);
2357                 btrfs_mark_buffer_dirty(leaf);
2358                 goto out;
2359         }
2360         btrfs_release_path(path);
2361
2362         ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode),
2363                         offset, 0, 0, end - offset, 0, end - offset, 0, 0, 0);
2364         if (ret)
2365                 return ret;
2366
2367 out:
2368         btrfs_release_path(path);
2369
2370         hole_em = alloc_extent_map();
2371         if (!hole_em) {
2372                 btrfs_drop_extent_cache(inode, offset, end - 1, 0);
2373                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
2374         } else {
2375                 hole_em->start = offset;
2376                 hole_em->len = end - offset;
2377                 hole_em->ram_bytes = hole_em->len;
2378                 hole_em->orig_start = offset;
2379
2380                 hole_em->block_start = EXTENT_MAP_HOLE;
2381                 hole_em->block_len = 0;
2382                 hole_em->orig_block_len = 0;
2383                 hole_em->bdev = fs_info->fs_devices->latest_bdev;
2384                 hole_em->compress_type = BTRFS_COMPRESS_NONE;
2385                 hole_em->generation = trans->transid;
2386
2387                 do {
2388                         btrfs_drop_extent_cache(inode, offset, end - 1, 0);
2389                         write_lock(&em_tree->lock);
2390                         ret = add_extent_mapping(em_tree, hole_em, 1);
2391                         write_unlock(&em_tree->lock);
2392                 } while (ret == -EEXIST);
2393                 free_extent_map(hole_em);
2394                 if (ret)
2395                         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2396                                         &inode->runtime_flags);
2397         }
2398
2399         return 0;
2400 }
2401
2402 /*
2403  * Find a hole extent on given inode and change start/len to the end of hole
2404  * extent.(hole/vacuum extent whose em->start <= start &&
2405  *         em->start + em->len > start)
2406  * When a hole extent is found, return 1 and modify start/len.
2407  */
2408 static int find_first_non_hole(struct inode *inode, u64 *start, u64 *len)
2409 {
2410         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2411         struct extent_map *em;
2412         int ret = 0;
2413
2414         em = btrfs_get_extent(BTRFS_I(inode), NULL, 0,
2415                               round_down(*start, fs_info->sectorsize),
2416                               round_up(*len, fs_info->sectorsize), 0);
2417         if (IS_ERR(em))
2418                 return PTR_ERR(em);
2419
2420         /* Hole or vacuum extent(only exists in no-hole mode) */
2421         if (em->block_start == EXTENT_MAP_HOLE) {
2422                 ret = 1;
2423                 *len = em->start + em->len > *start + *len ?
2424                        0 : *start + *len - em->start - em->len;
2425                 *start = em->start + em->len;
2426         }
2427         free_extent_map(em);
2428         return ret;
2429 }
2430
2431 static int btrfs_punch_hole_lock_range(struct inode *inode,
2432                                        const u64 lockstart,
2433                                        const u64 lockend,
2434                                        struct extent_state **cached_state)
2435 {
2436         while (1) {
2437                 struct btrfs_ordered_extent *ordered;
2438                 int ret;
2439
2440                 truncate_pagecache_range(inode, lockstart, lockend);
2441
2442                 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
2443                                  cached_state);
2444                 ordered = btrfs_lookup_first_ordered_extent(inode, lockend);
2445
2446                 /*
2447                  * We need to make sure we have no ordered extents in this range
2448                  * and nobody raced in and read a page in this range, if we did
2449                  * we need to try again.
2450                  */
2451                 if ((!ordered ||
2452                     (ordered->file_offset + ordered->len <= lockstart ||
2453                      ordered->file_offset > lockend)) &&
2454                      !filemap_range_has_page(inode->i_mapping,
2455                                              lockstart, lockend)) {
2456                         if (ordered)
2457                                 btrfs_put_ordered_extent(ordered);
2458                         break;
2459                 }
2460                 if (ordered)
2461                         btrfs_put_ordered_extent(ordered);
2462                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
2463                                      lockend, cached_state);
2464                 ret = btrfs_wait_ordered_range(inode, lockstart,
2465                                                lockend - lockstart + 1);
2466                 if (ret)
2467                         return ret;
2468         }
2469         return 0;
2470 }
2471
2472 static int btrfs_insert_clone_extent(struct btrfs_trans_handle *trans,
2473                                      struct inode *inode,
2474                                      struct btrfs_path *path,
2475                                      struct btrfs_clone_extent_info *clone_info,
2476                                      const u64 clone_len)
2477 {
2478         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2479         struct btrfs_root *root = BTRFS_I(inode)->root;
2480         struct btrfs_file_extent_item *extent;
2481         struct extent_buffer *leaf;
2482         struct btrfs_key key;
2483         int slot;
2484         struct btrfs_ref ref = { 0 };
2485         u64 ref_offset;
2486         int ret;
2487
2488         if (clone_len == 0)
2489                 return 0;
2490
2491         if (clone_info->disk_offset == 0 &&
2492             btrfs_fs_incompat(fs_info, NO_HOLES))
2493                 return 0;
2494
2495         key.objectid = btrfs_ino(BTRFS_I(inode));
2496         key.type = BTRFS_EXTENT_DATA_KEY;
2497         key.offset = clone_info->file_offset;
2498         ret = btrfs_insert_empty_item(trans, root, path, &key,
2499                                       clone_info->item_size);
2500         if (ret)
2501                 return ret;
2502         leaf = path->nodes[0];
2503         slot = path->slots[0];
2504         write_extent_buffer(leaf, clone_info->extent_buf,
2505                             btrfs_item_ptr_offset(leaf, slot),
2506                             clone_info->item_size);
2507         extent = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2508         btrfs_set_file_extent_offset(leaf, extent, clone_info->data_offset);
2509         btrfs_set_file_extent_num_bytes(leaf, extent, clone_len);
2510         btrfs_mark_buffer_dirty(leaf);
2511         btrfs_release_path(path);
2512
2513         /* If it's a hole, nothing more needs to be done. */
2514         if (clone_info->disk_offset == 0)
2515                 return 0;
2516
2517         inode_add_bytes(inode, clone_len);
2518         btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF,
2519                                clone_info->disk_offset,
2520                                clone_info->disk_len, 0);
2521         ref_offset = clone_info->file_offset - clone_info->data_offset;
2522         btrfs_init_data_ref(&ref, root->root_key.objectid,
2523                             btrfs_ino(BTRFS_I(inode)), ref_offset);
2524         ret = btrfs_inc_extent_ref(trans, &ref);
2525
2526         return ret;
2527 }
2528
2529 /*
2530  * The respective range must have been previously locked, as well as the inode.
2531  * The end offset is inclusive (last byte of the range).
2532  * @clone_info is NULL for fallocate's hole punching and non-NULL for extent
2533  * cloning.
2534  * When cloning, we don't want to end up in a state where we dropped extents
2535  * without inserting a new one, so we must abort the transaction to avoid a
2536  * corruption.
2537  */
2538 int btrfs_punch_hole_range(struct inode *inode, struct btrfs_path *path,
2539                            const u64 start, const u64 end,
2540                            struct btrfs_clone_extent_info *clone_info,
2541                            struct btrfs_trans_handle **trans_out)
2542 {
2543         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2544         u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1);
2545         u64 ino_size = round_up(inode->i_size, fs_info->sectorsize);
2546         struct btrfs_root *root = BTRFS_I(inode)->root;
2547         struct btrfs_trans_handle *trans = NULL;
2548         struct btrfs_block_rsv *rsv;
2549         unsigned int rsv_count;
2550         u64 cur_offset;
2551         u64 drop_end;
2552         u64 len = end - start;
2553         int ret = 0;
2554
2555         if (end <= start)
2556                 return -EINVAL;
2557
2558         rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
2559         if (!rsv) {
2560                 ret = -ENOMEM;
2561                 goto out;
2562         }
2563         rsv->size = btrfs_calc_insert_metadata_size(fs_info, 1);
2564         rsv->failfast = 1;
2565
2566         /*
2567          * 1 - update the inode
2568          * 1 - removing the extents in the range
2569          * 1 - adding the hole extent if no_holes isn't set or if we are cloning
2570          *     an extent
2571          */
2572         if (!btrfs_fs_incompat(fs_info, NO_HOLES) || clone_info)
2573                 rsv_count = 3;
2574         else
2575                 rsv_count = 2;
2576
2577         trans = btrfs_start_transaction(root, rsv_count);
2578         if (IS_ERR(trans)) {
2579                 ret = PTR_ERR(trans);
2580                 trans = NULL;
2581                 goto out_free;
2582         }
2583
2584         ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
2585                                       min_size, false);
2586         BUG_ON(ret);
2587         trans->block_rsv = rsv;
2588
2589         cur_offset = start;
2590         while (cur_offset < end) {
2591                 ret = __btrfs_drop_extents(trans, root, inode, path,
2592                                            cur_offset, end + 1, &drop_end,
2593                                            1, 0, 0, NULL);
2594                 if (ret != -ENOSPC) {
2595                         /*
2596                          * When cloning we want to avoid transaction aborts when
2597                          * nothing was done and we are attempting to clone parts
2598                          * of inline extents, in such cases -EOPNOTSUPP is
2599                          * returned by __btrfs_drop_extents() without having
2600                          * changed anything in the file.
2601                          */
2602                         if (clone_info && ret && ret != -EOPNOTSUPP)
2603                                 btrfs_abort_transaction(trans, ret);
2604                         break;
2605                 }
2606
2607                 trans->block_rsv = &fs_info->trans_block_rsv;
2608
2609                 if (!clone_info && cur_offset < drop_end &&
2610                     cur_offset < ino_size) {
2611                         ret = fill_holes(trans, BTRFS_I(inode), path,
2612                                         cur_offset, drop_end);
2613                         if (ret) {
2614                                 /*
2615                                  * If we failed then we didn't insert our hole
2616                                  * entries for the area we dropped, so now the
2617                                  * fs is corrupted, so we must abort the
2618                                  * transaction.
2619                                  */
2620                                 btrfs_abort_transaction(trans, ret);
2621                                 break;
2622                         }
2623                 }
2624
2625                 if (clone_info && drop_end > clone_info->file_offset) {
2626                         u64 clone_len = drop_end - clone_info->file_offset;
2627
2628                         ret = btrfs_insert_clone_extent(trans, inode, path,
2629                                                         clone_info, clone_len);
2630                         if (ret) {
2631                                 btrfs_abort_transaction(trans, ret);
2632                                 break;
2633                         }
2634                         clone_info->data_len -= clone_len;
2635                         clone_info->data_offset += clone_len;
2636                         clone_info->file_offset += clone_len;
2637                 }
2638
2639                 cur_offset = drop_end;
2640
2641                 ret = btrfs_update_inode(trans, root, inode);
2642                 if (ret)
2643                         break;
2644
2645                 btrfs_end_transaction(trans);
2646                 btrfs_btree_balance_dirty(fs_info);
2647
2648                 trans = btrfs_start_transaction(root, rsv_count);
2649                 if (IS_ERR(trans)) {
2650                         ret = PTR_ERR(trans);
2651                         trans = NULL;
2652                         break;
2653                 }
2654
2655                 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
2656                                               rsv, min_size, false);
2657                 BUG_ON(ret);    /* shouldn't happen */
2658                 trans->block_rsv = rsv;
2659
2660                 if (!clone_info) {
2661                         ret = find_first_non_hole(inode, &cur_offset, &len);
2662                         if (unlikely(ret < 0))
2663                                 break;
2664                         if (ret && !len) {
2665                                 ret = 0;
2666                                 break;
2667                         }
2668                 }
2669         }
2670
2671         /*
2672          * If we were cloning, force the next fsync to be a full one since we
2673          * we replaced (or just dropped in the case of cloning holes when
2674          * NO_HOLES is enabled) extents and extent maps.
2675          * This is for the sake of simplicity, and cloning into files larger
2676          * than 16Mb would force the full fsync any way (when
2677          * try_release_extent_mapping() is invoked during page cache truncation.
2678          */
2679         if (clone_info)
2680                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2681                         &BTRFS_I(inode)->runtime_flags);
2682
2683         if (ret)
2684                 goto out_trans;
2685
2686         trans->block_rsv = &fs_info->trans_block_rsv;
2687         /*
2688          * If we are using the NO_HOLES feature we might have had already an
2689          * hole that overlaps a part of the region [lockstart, lockend] and
2690          * ends at (or beyond) lockend. Since we have no file extent items to
2691          * represent holes, drop_end can be less than lockend and so we must
2692          * make sure we have an extent map representing the existing hole (the
2693          * call to __btrfs_drop_extents() might have dropped the existing extent
2694          * map representing the existing hole), otherwise the fast fsync path
2695          * will not record the existence of the hole region
2696          * [existing_hole_start, lockend].
2697          */
2698         if (drop_end <= end)
2699                 drop_end = end + 1;
2700         /*
2701          * Don't insert file hole extent item if it's for a range beyond eof
2702          * (because it's useless) or if it represents a 0 bytes range (when
2703          * cur_offset == drop_end).
2704          */
2705         if (!clone_info && cur_offset < ino_size && cur_offset < drop_end) {
2706                 ret = fill_holes(trans, BTRFS_I(inode), path,
2707                                 cur_offset, drop_end);
2708                 if (ret) {
2709                         /* Same comment as above. */
2710                         btrfs_abort_transaction(trans, ret);
2711                         goto out_trans;
2712                 }
2713         }
2714         if (clone_info) {
2715                 ret = btrfs_insert_clone_extent(trans, inode, path, clone_info,
2716                                                 clone_info->data_len);
2717                 if (ret) {
2718                         btrfs_abort_transaction(trans, ret);
2719                         goto out_trans;
2720                 }
2721         }
2722
2723 out_trans:
2724         if (!trans)
2725                 goto out_free;
2726
2727         trans->block_rsv = &fs_info->trans_block_rsv;
2728         if (ret)
2729                 btrfs_end_transaction(trans);
2730         else
2731                 *trans_out = trans;
2732 out_free:
2733         btrfs_free_block_rsv(fs_info, rsv);
2734 out:
2735         return ret;
2736 }
2737
2738 static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
2739 {
2740         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2741         struct btrfs_root *root = BTRFS_I(inode)->root;
2742         struct extent_state *cached_state = NULL;
2743         struct btrfs_path *path;
2744         struct btrfs_trans_handle *trans = NULL;
2745         u64 lockstart;
2746         u64 lockend;
2747         u64 tail_start;
2748         u64 tail_len;
2749         u64 orig_start = offset;
2750         int ret = 0;
2751         bool same_block;
2752         u64 ino_size;
2753         bool truncated_block = false;
2754         bool updated_inode = false;
2755
2756         ret = btrfs_wait_ordered_range(inode, offset, len);
2757         if (ret)
2758                 return ret;
2759
2760         inode_lock(inode);
2761         ino_size = round_up(inode->i_size, fs_info->sectorsize);
2762         ret = find_first_non_hole(inode, &offset, &len);
2763         if (ret < 0)
2764                 goto out_only_mutex;
2765         if (ret && !len) {
2766                 /* Already in a large hole */
2767                 ret = 0;
2768                 goto out_only_mutex;
2769         }
2770
2771         lockstart = round_up(offset, btrfs_inode_sectorsize(inode));
2772         lockend = round_down(offset + len,
2773                              btrfs_inode_sectorsize(inode)) - 1;
2774         same_block = (BTRFS_BYTES_TO_BLKS(fs_info, offset))
2775                 == (BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1));
2776         /*
2777          * We needn't truncate any block which is beyond the end of the file
2778          * because we are sure there is no data there.
2779          */
2780         /*
2781          * Only do this if we are in the same block and we aren't doing the
2782          * entire block.
2783          */
2784         if (same_block && len < fs_info->sectorsize) {
2785                 if (offset < ino_size) {
2786                         truncated_block = true;
2787                         ret = btrfs_truncate_block(inode, offset, len, 0);
2788                 } else {
2789                         ret = 0;
2790                 }
2791                 goto out_only_mutex;
2792         }
2793
2794         /* zero back part of the first block */
2795         if (offset < ino_size) {
2796                 truncated_block = true;
2797                 ret = btrfs_truncate_block(inode, offset, 0, 0);
2798                 if (ret) {
2799                         inode_unlock(inode);
2800                         return ret;
2801                 }
2802         }
2803
2804         /* Check the aligned pages after the first unaligned page,
2805          * if offset != orig_start, which means the first unaligned page
2806          * including several following pages are already in holes,
2807          * the extra check can be skipped */
2808         if (offset == orig_start) {
2809                 /* after truncate page, check hole again */
2810                 len = offset + len - lockstart;
2811                 offset = lockstart;
2812                 ret = find_first_non_hole(inode, &offset, &len);
2813                 if (ret < 0)
2814                         goto out_only_mutex;
2815                 if (ret && !len) {
2816                         ret = 0;
2817                         goto out_only_mutex;
2818                 }
2819                 lockstart = offset;
2820         }
2821
2822         /* Check the tail unaligned part is in a hole */
2823         tail_start = lockend + 1;
2824         tail_len = offset + len - tail_start;
2825         if (tail_len) {
2826                 ret = find_first_non_hole(inode, &tail_start, &tail_len);
2827                 if (unlikely(ret < 0))
2828                         goto out_only_mutex;
2829                 if (!ret) {
2830                         /* zero the front end of the last page */
2831                         if (tail_start + tail_len < ino_size) {
2832                                 truncated_block = true;
2833                                 ret = btrfs_truncate_block(inode,
2834                                                         tail_start + tail_len,
2835                                                         0, 1);
2836                                 if (ret)
2837                                         goto out_only_mutex;
2838                         }
2839                 }
2840         }
2841
2842         if (lockend < lockstart) {
2843                 ret = 0;
2844                 goto out_only_mutex;
2845         }
2846
2847         ret = btrfs_punch_hole_lock_range(inode, lockstart, lockend,
2848                                           &cached_state);
2849         if (ret)
2850                 goto out_only_mutex;
2851
2852         path = btrfs_alloc_path();
2853         if (!path) {
2854                 ret = -ENOMEM;
2855                 goto out;
2856         }
2857
2858         ret = btrfs_punch_hole_range(inode, path, lockstart, lockend, NULL,
2859                                      &trans);
2860         btrfs_free_path(path);
2861         if (ret)
2862                 goto out;
2863
2864         ASSERT(trans != NULL);
2865         inode_inc_iversion(inode);
2866         inode->i_mtime = inode->i_ctime = current_time(inode);
2867         ret = btrfs_update_inode(trans, root, inode);
2868         updated_inode = true;
2869         btrfs_end_transaction(trans);
2870         btrfs_btree_balance_dirty(fs_info);
2871 out:
2872         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
2873                              &cached_state);
2874 out_only_mutex:
2875         if (!updated_inode && truncated_block && !ret) {
2876                 /*
2877                  * If we only end up zeroing part of a page, we still need to
2878                  * update the inode item, so that all the time fields are
2879                  * updated as well as the necessary btrfs inode in memory fields
2880                  * for detecting, at fsync time, if the inode isn't yet in the
2881                  * log tree or it's there but not up to date.
2882                  */
2883                 struct timespec64 now = current_time(inode);
2884
2885                 inode_inc_iversion(inode);
2886                 inode->i_mtime = now;
2887                 inode->i_ctime = now;
2888                 trans = btrfs_start_transaction(root, 1);
2889                 if (IS_ERR(trans)) {
2890                         ret = PTR_ERR(trans);
2891                 } else {
2892                         int ret2;
2893
2894                         ret = btrfs_update_inode(trans, root, inode);
2895                         ret2 = btrfs_end_transaction(trans);
2896                         if (!ret)
2897                                 ret = ret2;
2898                 }
2899         }
2900         inode_unlock(inode);
2901         return ret;
2902 }
2903
2904 /* Helper structure to record which range is already reserved */
2905 struct falloc_range {
2906         struct list_head list;
2907         u64 start;
2908         u64 len;
2909 };
2910
2911 /*
2912  * Helper function to add falloc range
2913  *
2914  * Caller should have locked the larger range of extent containing
2915  * [start, len)
2916  */
2917 static int add_falloc_range(struct list_head *head, u64 start, u64 len)
2918 {
2919         struct falloc_range *prev = NULL;
2920         struct falloc_range *range = NULL;
2921
2922         if (list_empty(head))
2923                 goto insert;
2924
2925         /*
2926          * As fallocate iterate by bytenr order, we only need to check
2927          * the last range.
2928          */
2929         prev = list_entry(head->prev, struct falloc_range, list);
2930         if (prev->start + prev->len == start) {
2931                 prev->len += len;
2932                 return 0;
2933         }
2934 insert:
2935         range = kmalloc(sizeof(*range), GFP_KERNEL);
2936         if (!range)
2937                 return -ENOMEM;
2938         range->start = start;
2939         range->len = len;
2940         list_add_tail(&range->list, head);
2941         return 0;
2942 }
2943
2944 static int btrfs_fallocate_update_isize(struct inode *inode,
2945                                         const u64 end,
2946                                         const int mode)
2947 {
2948         struct btrfs_trans_handle *trans;
2949         struct btrfs_root *root = BTRFS_I(inode)->root;
2950         int ret;
2951         int ret2;
2952
2953         if (mode & FALLOC_FL_KEEP_SIZE || end <= i_size_read(inode))
2954                 return 0;
2955
2956         trans = btrfs_start_transaction(root, 1);
2957         if (IS_ERR(trans))
2958                 return PTR_ERR(trans);
2959
2960         inode->i_ctime = current_time(inode);
2961         i_size_write(inode, end);
2962         btrfs_ordered_update_i_size(inode, end, NULL);
2963         ret = btrfs_update_inode(trans, root, inode);
2964         ret2 = btrfs_end_transaction(trans);
2965
2966         return ret ? ret : ret2;
2967 }
2968
2969 enum {
2970         RANGE_BOUNDARY_WRITTEN_EXTENT,
2971         RANGE_BOUNDARY_PREALLOC_EXTENT,
2972         RANGE_BOUNDARY_HOLE,
2973 };
2974
2975 static int btrfs_zero_range_check_range_boundary(struct inode *inode,
2976                                                  u64 offset)
2977 {
2978         const u64 sectorsize = btrfs_inode_sectorsize(inode);
2979         struct extent_map *em;
2980         int ret;
2981
2982         offset = round_down(offset, sectorsize);
2983         em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
2984         if (IS_ERR(em))
2985                 return PTR_ERR(em);
2986
2987         if (em->block_start == EXTENT_MAP_HOLE)
2988                 ret = RANGE_BOUNDARY_HOLE;
2989         else if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2990                 ret = RANGE_BOUNDARY_PREALLOC_EXTENT;
2991         else
2992                 ret = RANGE_BOUNDARY_WRITTEN_EXTENT;
2993
2994         free_extent_map(em);
2995         return ret;
2996 }
2997
2998 static int btrfs_zero_range(struct inode *inode,
2999                             loff_t offset,
3000                             loff_t len,
3001                             const int mode)
3002 {
3003         struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
3004         struct extent_map *em;
3005         struct extent_changeset *data_reserved = NULL;
3006         int ret;
3007         u64 alloc_hint = 0;
3008         const u64 sectorsize = btrfs_inode_sectorsize(inode);
3009         u64 alloc_start = round_down(offset, sectorsize);
3010         u64 alloc_end = round_up(offset + len, sectorsize);
3011         u64 bytes_to_reserve = 0;
3012         bool space_reserved = false;
3013
3014         inode_dio_wait(inode);
3015
3016         em = btrfs_get_extent(BTRFS_I(inode), NULL, 0,
3017                               alloc_start, alloc_end - alloc_start, 0);
3018         if (IS_ERR(em)) {
3019                 ret = PTR_ERR(em);
3020                 goto out;
3021         }
3022
3023         /*
3024          * Avoid hole punching and extent allocation for some cases. More cases
3025          * could be considered, but these are unlikely common and we keep things
3026          * as simple as possible for now. Also, intentionally, if the target
3027          * range contains one or more prealloc extents together with regular
3028          * extents and holes, we drop all the existing extents and allocate a
3029          * new prealloc extent, so that we get a larger contiguous disk extent.
3030          */
3031         if (em->start <= alloc_start &&
3032             test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
3033                 const u64 em_end = em->start + em->len;
3034
3035                 if (em_end >= offset + len) {
3036                         /*
3037                          * The whole range is already a prealloc extent,
3038                          * do nothing except updating the inode's i_size if
3039                          * needed.
3040                          */
3041                         free_extent_map(em);
3042                         ret = btrfs_fallocate_update_isize(inode, offset + len,
3043                                                            mode);
3044                         goto out;
3045                 }
3046                 /*
3047                  * Part of the range is already a prealloc extent, so operate
3048                  * only on the remaining part of the range.
3049                  */
3050                 alloc_start = em_end;
3051                 ASSERT(IS_ALIGNED(alloc_start, sectorsize));
3052                 len = offset + len - alloc_start;
3053                 offset = alloc_start;
3054                 alloc_hint = em->block_start + em->len;
3055         }
3056         free_extent_map(em);
3057
3058         if (BTRFS_BYTES_TO_BLKS(fs_info, offset) ==
3059             BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1)) {
3060                 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0,
3061                                       alloc_start, sectorsize, 0);
3062                 if (IS_ERR(em)) {
3063                         ret = PTR_ERR(em);
3064                         goto out;
3065                 }
3066
3067                 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
3068                         free_extent_map(em);
3069                         ret = btrfs_fallocate_update_isize(inode, offset + len,
3070                                                            mode);
3071                         goto out;
3072                 }
3073                 if (len < sectorsize && em->block_start != EXTENT_MAP_HOLE) {
3074                         free_extent_map(em);
3075                         ret = btrfs_truncate_block(inode, offset, len, 0);
3076                         if (!ret)
3077                                 ret = btrfs_fallocate_update_isize(inode,
3078                                                                    offset + len,
3079                                                                    mode);
3080                         return ret;
3081                 }
3082                 free_extent_map(em);
3083                 alloc_start = round_down(offset, sectorsize);
3084                 alloc_end = alloc_start + sectorsize;
3085                 goto reserve_space;
3086         }
3087
3088         alloc_start = round_up(offset, sectorsize);
3089         alloc_end = round_down(offset + len, sectorsize);
3090
3091         /*
3092          * For unaligned ranges, check the pages at the boundaries, they might
3093          * map to an extent, in which case we need to partially zero them, or
3094          * they might map to a hole, in which case we need our allocation range
3095          * to cover them.
3096          */
3097         if (!IS_ALIGNED(offset, sectorsize)) {
3098                 ret = btrfs_zero_range_check_range_boundary(inode, offset);
3099                 if (ret < 0)
3100                         goto out;
3101                 if (ret == RANGE_BOUNDARY_HOLE) {
3102                         alloc_start = round_down(offset, sectorsize);
3103                         ret = 0;
3104                 } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3105                         ret = btrfs_truncate_block(inode, offset, 0, 0);
3106                         if (ret)
3107                                 goto out;
3108                 } else {
3109                         ret = 0;
3110                 }
3111         }
3112
3113         if (!IS_ALIGNED(offset + len, sectorsize)) {
3114                 ret = btrfs_zero_range_check_range_boundary(inode,
3115                                                             offset + len);
3116                 if (ret < 0)
3117                         goto out;
3118                 if (ret == RANGE_BOUNDARY_HOLE) {
3119                         alloc_end = round_up(offset + len, sectorsize);
3120                         ret = 0;
3121                 } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3122                         ret = btrfs_truncate_block(inode, offset + len, 0, 1);
3123                         if (ret)
3124                                 goto out;
3125                 } else {
3126                         ret = 0;
3127                 }
3128         }
3129
3130 reserve_space:
3131         if (alloc_start < alloc_end) {
3132                 struct extent_state *cached_state = NULL;
3133                 const u64 lockstart = alloc_start;
3134                 const u64 lockend = alloc_end - 1;
3135
3136                 bytes_to_reserve = alloc_end - alloc_start;
3137                 ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3138                                                       bytes_to_reserve);
3139                 if (ret < 0)
3140                         goto out;
3141                 space_reserved = true;
3142                 ret = btrfs_punch_hole_lock_range(inode, lockstart, lockend,
3143                                                   &cached_state);
3144                 if (ret)
3145                         goto out;
3146                 ret = btrfs_qgroup_reserve_data(BTRFS_I(inode), &data_reserved,
3147                                                 alloc_start, bytes_to_reserve);
3148                 if (ret) {
3149                         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
3150                                              lockend, &cached_state);
3151                         goto out;
3152                 }
3153                 ret = btrfs_prealloc_file_range(inode, mode, alloc_start,
3154                                                 alloc_end - alloc_start,
3155                                                 i_blocksize(inode),
3156                                                 offset + len, &alloc_hint);
3157                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
3158                                      lockend, &cached_state);
3159                 /* btrfs_prealloc_file_range releases reserved space on error */
3160                 if (ret) {
3161                         space_reserved = false;
3162                         goto out;
3163                 }
3164         }
3165         ret = btrfs_fallocate_update_isize(inode, offset + len, mode);
3166  out:
3167         if (ret && space_reserved)
3168                 btrfs_free_reserved_data_space(inode, data_reserved,
3169                                                alloc_start, bytes_to_reserve);
3170         extent_changeset_free(data_reserved);
3171
3172         return ret;
3173 }
3174
3175 static long btrfs_fallocate(struct file *file, int mode,
3176                             loff_t offset, loff_t len)
3177 {
3178         struct inode *inode = file_inode(file);
3179         struct extent_state *cached_state = NULL;
3180         struct extent_changeset *data_reserved = NULL;
3181         struct falloc_range *range;
3182         struct falloc_range *tmp;
3183         struct list_head reserve_list;
3184         u64 cur_offset;
3185         u64 last_byte;
3186         u64 alloc_start;
3187         u64 alloc_end;
3188         u64 alloc_hint = 0;
3189         u64 locked_end;
3190         u64 actual_end = 0;
3191         struct extent_map *em;
3192         int blocksize = btrfs_inode_sectorsize(inode);
3193         int ret;
3194
3195         alloc_start = round_down(offset, blocksize);
3196         alloc_end = round_up(offset + len, blocksize);
3197         cur_offset = alloc_start;
3198
3199         /* Make sure we aren't being give some crap mode */
3200         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
3201                      FALLOC_FL_ZERO_RANGE))
3202                 return -EOPNOTSUPP;
3203
3204         if (mode & FALLOC_FL_PUNCH_HOLE)
3205                 return btrfs_punch_hole(inode, offset, len);
3206
3207         /*
3208          * Only trigger disk allocation, don't trigger qgroup reserve
3209          *
3210          * For qgroup space, it will be checked later.
3211          */
3212         if (!(mode & FALLOC_FL_ZERO_RANGE)) {
3213                 ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3214                                                       alloc_end - alloc_start);
3215                 if (ret < 0)
3216                         return ret;
3217         }
3218
3219         inode_lock(inode);
3220
3221         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) {
3222                 ret = inode_newsize_ok(inode, offset + len);
3223                 if (ret)
3224                         goto out;
3225         }
3226
3227         /*
3228          * TODO: Move these two operations after we have checked
3229          * accurate reserved space, or fallocate can still fail but
3230          * with page truncated or size expanded.
3231          *
3232          * But that's a minor problem and won't do much harm BTW.
3233          */
3234         if (alloc_start > inode->i_size) {
3235                 ret = btrfs_cont_expand(inode, i_size_read(inode),
3236                                         alloc_start);
3237                 if (ret)
3238                         goto out;
3239         } else if (offset + len > inode->i_size) {
3240                 /*
3241                  * If we are fallocating from the end of the file onward we
3242                  * need to zero out the end of the block if i_size lands in the
3243                  * middle of a block.
3244                  */
3245                 ret = btrfs_truncate_block(inode, inode->i_size, 0, 0);
3246                 if (ret)
3247                         goto out;
3248         }
3249
3250         /*
3251          * wait for ordered IO before we have any locks.  We'll loop again
3252          * below with the locks held.
3253          */
3254         ret = btrfs_wait_ordered_range(inode, alloc_start,
3255                                        alloc_end - alloc_start);
3256         if (ret)
3257                 goto out;
3258
3259         if (mode & FALLOC_FL_ZERO_RANGE) {
3260                 ret = btrfs_zero_range(inode, offset, len, mode);
3261                 inode_unlock(inode);
3262                 return ret;
3263         }
3264
3265         locked_end = alloc_end - 1;
3266         while (1) {
3267                 struct btrfs_ordered_extent *ordered;
3268
3269                 /* the extent lock is ordered inside the running
3270                  * transaction
3271                  */
3272                 lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
3273                                  locked_end, &cached_state);
3274                 ordered = btrfs_lookup_first_ordered_extent(inode, locked_end);
3275
3276                 if (ordered &&
3277                     ordered->file_offset + ordered->len > alloc_start &&
3278                     ordered->file_offset < alloc_end) {
3279                         btrfs_put_ordered_extent(ordered);
3280                         unlock_extent_cached(&BTRFS_I(inode)->io_tree,
3281                                              alloc_start, locked_end,
3282                                              &cached_state);
3283                         /*
3284                          * we can't wait on the range with the transaction
3285                          * running or with the extent lock held
3286                          */
3287                         ret = btrfs_wait_ordered_range(inode, alloc_start,
3288                                                        alloc_end - alloc_start);
3289                         if (ret)
3290                                 goto out;
3291                 } else {
3292                         if (ordered)
3293                                 btrfs_put_ordered_extent(ordered);
3294                         break;
3295                 }
3296         }
3297
3298         /* First, check if we exceed the qgroup limit */
3299         INIT_LIST_HEAD(&reserve_list);
3300         while (cur_offset < alloc_end) {
3301                 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
3302                                       alloc_end - cur_offset, 0);
3303                 if (IS_ERR(em)) {
3304                         ret = PTR_ERR(em);
3305                         break;
3306                 }
3307                 last_byte = min(extent_map_end(em), alloc_end);
3308                 actual_end = min_t(u64, extent_map_end(em), offset + len);
3309                 last_byte = ALIGN(last_byte, blocksize);
3310                 if (em->block_start == EXTENT_MAP_HOLE ||
3311                     (cur_offset >= inode->i_size &&
3312                      !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
3313                         ret = add_falloc_range(&reserve_list, cur_offset,
3314                                                last_byte - cur_offset);
3315                         if (ret < 0) {
3316                                 free_extent_map(em);
3317                                 break;
3318                         }
3319                         ret = btrfs_qgroup_reserve_data(BTRFS_I(inode),
3320                                         &data_reserved, cur_offset,
3321                                         last_byte - cur_offset);
3322                         if (ret < 0) {
3323                                 cur_offset = last_byte;
3324                                 free_extent_map(em);
3325                                 break;
3326                         }
3327                 } else {
3328                         /*
3329                          * Do not need to reserve unwritten extent for this
3330                          * range, free reserved data space first, otherwise
3331                          * it'll result in false ENOSPC error.
3332                          */
3333                         btrfs_free_reserved_data_space(inode, data_reserved,
3334                                         cur_offset, last_byte - cur_offset);
3335                 }
3336                 free_extent_map(em);
3337                 cur_offset = last_byte;
3338         }
3339
3340         /*
3341          * If ret is still 0, means we're OK to fallocate.
3342          * Or just cleanup the list and exit.
3343          */
3344         list_for_each_entry_safe(range, tmp, &reserve_list, list) {
3345                 if (!ret)
3346                         ret = btrfs_prealloc_file_range(inode, mode,
3347                                         range->start,
3348                                         range->len, i_blocksize(inode),
3349                                         offset + len, &alloc_hint);
3350                 else
3351                         btrfs_free_reserved_data_space(inode,
3352                                         data_reserved, range->start,
3353                                         range->len);
3354                 list_del(&range->list);
3355                 kfree(range);
3356         }
3357         if (ret < 0)
3358                 goto out_unlock;
3359
3360         /*
3361          * We didn't need to allocate any more space, but we still extended the
3362          * size of the file so we need to update i_size and the inode item.
3363          */
3364         ret = btrfs_fallocate_update_isize(inode, actual_end, mode);
3365 out_unlock:
3366         unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
3367                              &cached_state);
3368 out:
3369         inode_unlock(inode);
3370         /* Let go of our reservation. */
3371         if (ret != 0 && !(mode & FALLOC_FL_ZERO_RANGE))
3372                 btrfs_free_reserved_data_space(inode, data_reserved,
3373                                 cur_offset, alloc_end - cur_offset);
3374         extent_changeset_free(data_reserved);
3375         return ret;
3376 }
3377
3378 static int find_desired_extent(struct inode *inode, loff_t *offset, int whence)
3379 {
3380         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3381         struct extent_map *em = NULL;
3382         struct extent_state *cached_state = NULL;
3383         u64 lockstart;
3384         u64 lockend;
3385         u64 start;
3386         u64 len;
3387         int ret = 0;
3388
3389         if (inode->i_size == 0)
3390                 return -ENXIO;
3391
3392         /*
3393          * *offset can be negative, in this case we start finding DATA/HOLE from
3394          * the very start of the file.
3395          */
3396         start = max_t(loff_t, 0, *offset);
3397
3398         lockstart = round_down(start, fs_info->sectorsize);
3399         lockend = round_up(i_size_read(inode),
3400                            fs_info->sectorsize);
3401         if (lockend <= lockstart)
3402                 lockend = lockstart + fs_info->sectorsize;
3403         lockend--;
3404         len = lockend - lockstart + 1;
3405
3406         lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
3407                          &cached_state);
3408
3409         while (start < inode->i_size) {
3410                 em = btrfs_get_extent_fiemap(BTRFS_I(inode), start, len);
3411                 if (IS_ERR(em)) {
3412                         ret = PTR_ERR(em);
3413                         em = NULL;
3414                         break;
3415                 }
3416
3417                 if (whence == SEEK_HOLE &&
3418                     (em->block_start == EXTENT_MAP_HOLE ||
3419                      test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
3420                         break;
3421                 else if (whence == SEEK_DATA &&
3422                            (em->block_start != EXTENT_MAP_HOLE &&
3423                             !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
3424                         break;
3425
3426                 start = em->start + em->len;
3427                 free_extent_map(em);
3428                 em = NULL;
3429                 cond_resched();
3430         }
3431         free_extent_map(em);
3432         if (!ret) {
3433                 if (whence == SEEK_DATA && start >= inode->i_size)
3434                         ret = -ENXIO;
3435                 else
3436                         *offset = min_t(loff_t, start, inode->i_size);
3437         }
3438         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
3439                              &cached_state);
3440         return ret;
3441 }
3442
3443 static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence)
3444 {
3445         struct inode *inode = file->f_mapping->host;
3446         int ret;
3447
3448         inode_lock(inode);
3449         switch (whence) {
3450         case SEEK_END:
3451         case SEEK_CUR:
3452                 offset = generic_file_llseek(file, offset, whence);
3453                 goto out;
3454         case SEEK_DATA:
3455         case SEEK_HOLE:
3456                 if (offset >= i_size_read(inode)) {
3457                         inode_unlock(inode);
3458                         return -ENXIO;
3459                 }
3460
3461                 ret = find_desired_extent(inode, &offset, whence);
3462                 if (ret) {
3463                         inode_unlock(inode);
3464                         return ret;
3465                 }
3466         }
3467
3468         offset = vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
3469 out:
3470         inode_unlock(inode);
3471         return offset;
3472 }
3473
3474 static int btrfs_file_open(struct inode *inode, struct file *filp)
3475 {
3476         filp->f_mode |= FMODE_NOWAIT;
3477         return generic_file_open(inode, filp);
3478 }
3479
3480 const struct file_operations btrfs_file_operations = {
3481         .llseek         = btrfs_file_llseek,
3482         .read_iter      = generic_file_read_iter,
3483         .splice_read    = generic_file_splice_read,
3484         .write_iter     = btrfs_file_write_iter,
3485         .mmap           = btrfs_file_mmap,
3486         .open           = btrfs_file_open,
3487         .release        = btrfs_release_file,
3488         .fsync          = btrfs_sync_file,
3489         .fallocate      = btrfs_fallocate,
3490         .unlocked_ioctl = btrfs_ioctl,
3491 #ifdef CONFIG_COMPAT
3492         .compat_ioctl   = btrfs_compat_ioctl,
3493 #endif
3494         .remap_file_range = btrfs_remap_file_range,
3495 };
3496
3497 void __cold btrfs_auto_defrag_exit(void)
3498 {
3499         kmem_cache_destroy(btrfs_inode_defrag_cachep);
3500 }
3501
3502 int __init btrfs_auto_defrag_init(void)
3503 {
3504         btrfs_inode_defrag_cachep = kmem_cache_create("btrfs_inode_defrag",
3505                                         sizeof(struct inode_defrag), 0,
3506                                         SLAB_MEM_SPREAD,
3507                                         NULL);
3508         if (!btrfs_inode_defrag_cachep)
3509                 return -ENOMEM;
3510
3511         return 0;
3512 }
3513
3514 int btrfs_fdatawrite_range(struct inode *inode, loff_t start, loff_t end)
3515 {
3516         int ret;
3517
3518         /*
3519          * So with compression we will find and lock a dirty page and clear the
3520          * first one as dirty, setup an async extent, and immediately return
3521          * with the entire range locked but with nobody actually marked with
3522          * writeback.  So we can't just filemap_write_and_wait_range() and
3523          * expect it to work since it will just kick off a thread to do the
3524          * actual work.  So we need to call filemap_fdatawrite_range _again_
3525          * since it will wait on the page lock, which won't be unlocked until
3526          * after the pages have been marked as writeback and so we're good to go
3527          * from there.  We have to do this otherwise we'll miss the ordered
3528          * extents and that results in badness.  Please Josef, do not think you
3529          * know better and pull this out at some point in the future, it is
3530          * right and you are wrong.
3531          */
3532         ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
3533         if (!ret && test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
3534                              &BTRFS_I(inode)->runtime_flags))
3535                 ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
3536
3537         return ret;
3538 }