fs/btrfs/inode.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/kernel.h>
   7 #include <linux/bio.h>
   8 #include <linux/buffer_head.h>
   9 #include <linux/file.h>
  10 #include <linux/fs.h>
  11 #include <linux/pagemap.h>
  12 #include <linux/highmem.h>
  13 #include <linux/time.h>
  14 #include <linux/init.h>
  15 #include <linux/string.h>
  16 #include <linux/backing-dev.h>
  17 #include <linux/writeback.h>
  18 #include <linux/compat.h>
  19 #include <linux/xattr.h>
  20 #include <linux/posix_acl.h>
  21 #include <linux/falloc.h>
  22 #include <linux/slab.h>
  23 #include <linux/ratelimit.h>
  24 #include <linux/btrfs.h>
  25 #include <linux/blkdev.h>
  26 #include <linux/posix_acl_xattr.h>
  27 #include <linux/uio.h>
  28 #include <linux/magic.h>
  29 #include <linux/iversion.h>
  30 #include <linux/swap.h>
  31 #include <linux/sched/mm.h>
  32 #include <asm/unaligned.h>
  33 #include "misc.h"
  34 #include "ctree.h"
  35 #include "disk-io.h"
  36 #include "transaction.h"
  37 #include "btrfs_inode.h"
  38 #include "print-tree.h"
  39 #include "ordered-data.h"
  40 #include "xattr.h"
  41 #include "tree-log.h"
  42 #include "volumes.h"
  43 #include "compression.h"
  44 #include "locking.h"
  45 #include "free-space-cache.h"
  46 #include "inode-map.h"
  47 #include "backref.h"
  48 #include "props.h"
  49 #include "qgroup.h"
  50 #include "delalloc-space.h"
  51 #include "block-group.h"
  52 #include "space-info.h"
  53
  54 struct btrfs_iget_args {
  55         struct btrfs_key *location;
  56         struct btrfs_root *root;
  57 };
  58
  59 struct btrfs_dio_data {
  60         u64 reserve;
  61         u64 unsubmitted_oe_range_start;
  62         u64 unsubmitted_oe_range_end;
  63         int overwrite;
  64 };
  65
  66 static const struct inode_operations btrfs_dir_inode_operations;
  67 static const struct inode_operations btrfs_symlink_inode_operations;
  68 static const struct inode_operations btrfs_dir_ro_inode_operations;
  69 static const struct inode_operations btrfs_special_inode_operations;
  70 static const struct inode_operations btrfs_file_inode_operations;
  71 static const struct address_space_operations btrfs_aops;
  72 static const struct file_operations btrfs_dir_file_operations;
  73 static const struct extent_io_ops btrfs_extent_io_ops;
  74
  75 static struct kmem_cache *btrfs_inode_cachep;
  76 struct kmem_cache *btrfs_trans_handle_cachep;
  77 struct kmem_cache *btrfs_path_cachep;
  78 struct kmem_cache *btrfs_free_space_cachep;
  79 struct kmem_cache *btrfs_free_space_bitmap_cachep;
  80
  81 static int btrfs_setsize(struct inode *inode, struct iattr *attr);
  82 static int btrfs_truncate(struct inode *inode, bool skip_writeback);
  83 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
  84 static noinline int cow_file_range(struct inode *inode,
  85                                    struct page *locked_page,
  86                                    u64 start, u64 end, int *page_started,
  87                                    unsigned long *nr_written, int unlock);
  88 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len,
  89                                        u64 orig_start, u64 block_start,
  90                                        u64 block_len, u64 orig_block_len,
  91                                        u64 ram_bytes, int compress_type,
  92                                        int type);
  93
  94 static void __endio_write_update_ordered(struct inode *inode,
  95                                          const u64 offset, const u64 bytes,
  96                                          const bool uptodate);
  97
  98 /*
  99  * Cleanup all submitted ordered extents in specified range to handle errors
 100  * from the btrfs_run_delalloc_range() callback.
 101  *
 102  * NOTE: caller must ensure that when an error happens, it can not call
 103  * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
 104  * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
 105  * to be released, which we want to happen only when finishing the ordered
 106  * extent (btrfs_finish_ordered_io()).
 107  */
 108 static inline void btrfs_cleanup_ordered_extents(struct inode *inode,
 109                                                  struct page *locked_page,
 110                                                  u64 offset, u64 bytes)
 111 {
 112         unsigned long index = offset >> PAGE_SHIFT;
 113         unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT;
 114         u64 page_start = page_offset(locked_page);
 115         u64 page_end = page_start + PAGE_SIZE - 1;
 116
 117         struct page *page;
 118
 119         while (index <= end_index) {
 120                 page = find_get_page(inode->i_mapping, index);
 121                 index++;
 122                 if (!page)
 123                         continue;
 124                 ClearPagePrivate2(page);
 125                 put_page(page);
 126         }
 127
 128         /*
 129          * In case this page belongs to the delalloc range being instantiated
 130          * then skip it, since the first page of a range is going to be
 131          * properly cleaned up by the caller of run_delalloc_range
 132          */
 133         if (page_start >= offset && page_end <= (offset + bytes - 1)) {
 134                 offset += PAGE_SIZE;
 135                 bytes -= PAGE_SIZE;
 136         }
 137
 138         return __endio_write_update_ordered(inode, offset, bytes, false);
 139 }
 140
 141 static int btrfs_dirty_inode(struct inode *inode);
 142
 143 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
 144 void btrfs_test_inode_set_ops(struct inode *inode)
 145 {
 146         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
 147 }
 148 #endif
 149
 150 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
 151                                      struct inode *inode,  struct inode *dir,
 152                                      const struct qstr *qstr)
 153 {
 154         int err;
 155
 156         err = btrfs_init_acl(trans, inode, dir);
 157         if (!err)
 158                 err = btrfs_xattr_security_init(trans, inode, dir, qstr);
 159         return err;
 160 }
 161
 162 /*
 163  * this does all the hard work for inserting an inline extent into
 164  * the btree.  The caller should have done a btrfs_drop_extents so that
 165  * no overlapping inline items exist in the btree
 166  */
 167 static int insert_inline_extent(struct btrfs_trans_handle *trans,
 168                                 struct btrfs_path *path, int extent_inserted,
 169                                 struct btrfs_root *root, struct inode *inode,
 170                                 u64 start, size_t size, size_t compressed_size,
 171                                 int compress_type,
 172                                 struct page **compressed_pages)
 173 {
 174         struct extent_buffer *leaf;
 175         struct page *page = NULL;
 176         char *kaddr;
 177         unsigned long ptr;
 178         struct btrfs_file_extent_item *ei;
 179         int ret;
 180         size_t cur_size = size;
 181         unsigned long offset;
 182
 183         ASSERT((compressed_size > 0 && compressed_pages) ||
 184                (compressed_size == 0 && !compressed_pages));
 185
 186         if (compressed_size && compressed_pages)
 187                 cur_size = compressed_size;
 188
 189         inode_add_bytes(inode, size);
 190
 191         if (!extent_inserted) {
 192                 struct btrfs_key key;
 193                 size_t datasize;
 194
 195                 key.objectid = btrfs_ino(BTRFS_I(inode));
 196                 key.offset = start;
 197                 key.type = BTRFS_EXTENT_DATA_KEY;
 198
 199                 datasize = btrfs_file_extent_calc_inline_size(cur_size);
 200                 path->leave_spinning = 1;
 201                 ret = btrfs_insert_empty_item(trans, root, path, &key,
 202                                               datasize);
 203                 if (ret)
 204                         goto fail;
 205         }
 206         leaf = path->nodes[0];
 207         ei = btrfs_item_ptr(leaf, path->slots[0],
 208                             struct btrfs_file_extent_item);
 209         btrfs_set_file_extent_generation(leaf, ei, trans->transid);
 210         btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
 211         btrfs_set_file_extent_encryption(leaf, ei, 0);
 212         btrfs_set_file_extent_other_encoding(leaf, ei, 0);
 213         btrfs_set_file_extent_ram_bytes(leaf, ei, size);
 214         ptr = btrfs_file_extent_inline_start(ei);
 215
 216         if (compress_type != BTRFS_COMPRESS_NONE) {
 217                 struct page *cpage;
 218                 int i = 0;
 219                 while (compressed_size > 0) {
 220                         cpage = compressed_pages[i];
 221                         cur_size = min_t(unsigned long, compressed_size,
 222                                        PAGE_SIZE);
 223
 224                         kaddr = kmap_atomic(cpage);
 225                         write_extent_buffer(leaf, kaddr, ptr, cur_size);
 226                         kunmap_atomic(kaddr);
 227
 228                         i++;
 229                         ptr += cur_size;
 230                         compressed_size -= cur_size;
 231                 }
 232                 btrfs_set_file_extent_compression(leaf, ei,
 233                                                   compress_type);
 234         } else {
 235                 page = find_get_page(inode->i_mapping,
 236                                      start >> PAGE_SHIFT);
 237                 btrfs_set_file_extent_compression(leaf, ei, 0);
 238                 kaddr = kmap_atomic(page);
 239                 offset = offset_in_page(start);
 240                 write_extent_buffer(leaf, kaddr + offset, ptr, size);
 241                 kunmap_atomic(kaddr);
 242                 put_page(page);
 243         }
 244         btrfs_mark_buffer_dirty(leaf);
 245         btrfs_release_path(path);
 246
 247         /*
 248          * we're an inline extent, so nobody can
 249          * extend the file past i_size without locking
 250          * a page we already have locked.
 251          *
 252          * We must do any isize and inode updates
 253          * before we unlock the pages.  Otherwise we
 254          * could end up racing with unlink.
 255          */
 256         BTRFS_I(inode)->disk_i_size = inode->i_size;
 257         ret = btrfs_update_inode(trans, root, inode);
 258
 259 fail:
 260         return ret;
 261 }
 262
 263
 264 /*
 265  * conditionally insert an inline extent into the file.  This
 266  * does the checks required to make sure the data is small enough
 267  * to fit as an inline extent.
 268  */
 269 static noinline int cow_file_range_inline(struct inode *inode, u64 start,
 270                                           u64 end, size_t compressed_size,
 271                                           int compress_type,
 272                                           struct page **compressed_pages)
 273 {
 274         struct btrfs_root *root = BTRFS_I(inode)->root;
 275         struct btrfs_fs_info *fs_info = root->fs_info;
 276         struct btrfs_trans_handle *trans;
 277         u64 isize = i_size_read(inode);
 278         u64 actual_end = min(end + 1, isize);
 279         u64 inline_len = actual_end - start;
 280         u64 aligned_end = ALIGN(end, fs_info->sectorsize);
 281         u64 data_len = inline_len;
 282         int ret;
 283         struct btrfs_path *path;
 284         int extent_inserted = 0;
 285         u32 extent_item_size;
 286
 287         if (compressed_size)
 288                 data_len = compressed_size;
 289
 290         if (start > 0 ||
 291             actual_end > fs_info->sectorsize ||
 292             data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) ||
 293             (!compressed_size &&
 294             (actual_end & (fs_info->sectorsize - 1)) == 0) ||
 295             end + 1 < isize ||
 296             data_len > fs_info->max_inline) {
 297                 return 1;
 298         }
 299
 300         path = btrfs_alloc_path();
 301         if (!path)
 302                 return -ENOMEM;
 303
 304         trans = btrfs_join_transaction(root);
 305         if (IS_ERR(trans)) {
 306                 btrfs_free_path(path);
 307                 return PTR_ERR(trans);
 308         }
 309         trans->block_rsv = &BTRFS_I(inode)->block_rsv;
 310
 311         if (compressed_size && compressed_pages)
 312                 extent_item_size = btrfs_file_extent_calc_inline_size(
 313                    compressed_size);
 314         else
 315                 extent_item_size = btrfs_file_extent_calc_inline_size(
 316                     inline_len);
 317
 318         ret = __btrfs_drop_extents(trans, root, inode, path,
 319                                    start, aligned_end, NULL,
 320                                    1, 1, extent_item_size, &extent_inserted);
 321         if (ret) {
 322                 btrfs_abort_transaction(trans, ret);
 323                 goto out;
 324         }
 325
 326         if (isize > actual_end)
 327                 inline_len = min_t(u64, isize, actual_end);
 328         ret = insert_inline_extent(trans, path, extent_inserted,
 329                                    root, inode, start,
 330                                    inline_len, compressed_size,
 331                                    compress_type, compressed_pages);
 332         if (ret && ret != -ENOSPC) {
 333                 btrfs_abort_transaction(trans, ret);
 334                 goto out;
 335         } else if (ret == -ENOSPC) {
 336                 ret = 1;
 337                 goto out;
 338         }
 339
 340         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
 341         btrfs_drop_extent_cache(BTRFS_I(inode), start, aligned_end - 1, 0);
 342 out:
 343         /*
 344          * Don't forget to free the reserved space, as for inlined extent
 345          * it won't count as data extent, free them directly here.
 346          * And at reserve time, it's always aligned to page size, so
 347          * just free one page here.
 348          */
 349         btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE);
 350         btrfs_free_path(path);
 351         btrfs_end_transaction(trans);
 352         return ret;
 353 }
 354
 355 struct async_extent {
 356         u64 start;
 357         u64 ram_size;
 358         u64 compressed_size;
 359         struct page **pages;
 360         unsigned long nr_pages;
 361         int compress_type;
 362         struct list_head list;
 363 };
 364
 365 struct async_chunk {
 366         struct inode *inode;
 367         struct page *locked_page;
 368         u64 start;
 369         u64 end;
 370         unsigned int write_flags;
 371         struct list_head extents;
 372         struct btrfs_work work;
 373         atomic_t *pending;
 374 };
 375
 376 struct async_cow {
 377         /* Number of chunks in flight; must be first in the structure */
 378         atomic_t num_chunks;
 379         struct async_chunk chunks[];
 380 };
 381
 382 static noinline int add_async_extent(struct async_chunk *cow,
 383                                      u64 start, u64 ram_size,
 384                                      u64 compressed_size,
 385                                      struct page **pages,
 386                                      unsigned long nr_pages,
 387                                      int compress_type)
 388 {
 389         struct async_extent *async_extent;
 390
 391         async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
 392         BUG_ON(!async_extent); /* -ENOMEM */
 393         async_extent->start = start;
 394         async_extent->ram_size = ram_size;
 395         async_extent->compressed_size = compressed_size;
 396         async_extent->pages = pages;
 397         async_extent->nr_pages = nr_pages;
 398         async_extent->compress_type = compress_type;
 399         list_add_tail(&async_extent->list, &cow->extents);
 400         return 0;
 401 }
 402
 403 /*
 404  * Check if the inode has flags compatible with compression
 405  */
 406 static inline bool inode_can_compress(struct inode *inode)
 407 {
 408         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW ||
 409             BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
 410                 return false;
 411         return true;
 412 }
 413
 414 /*
 415  * Check if the inode needs to be submitted to compression, based on mount
 416  * options, defragmentation, properties or heuristics.
 417  */
 418 static inline int inode_need_compress(struct inode *inode, u64 start, u64 end)
 419 {
 420         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 421
 422         if (!inode_can_compress(inode)) {
 423                 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
 424                         KERN_ERR "BTRFS: unexpected compression for ino %llu\n",
 425                         btrfs_ino(BTRFS_I(inode)));
 426                 return 0;
 427         }
 428         /* force compress */
 429         if (btrfs_test_opt(fs_info, FORCE_COMPRESS))
 430                 return 1;
 431         /* defrag ioctl */
 432         if (BTRFS_I(inode)->defrag_compress)
 433                 return 1;
 434         /* bad compression ratios */
 435         if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS)
 436                 return 0;
 437         if (btrfs_test_opt(fs_info, COMPRESS) ||
 438             BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS ||
 439             BTRFS_I(inode)->prop_compress)
 440                 return btrfs_compress_heuristic(inode, start, end);
 441         return 0;
 442 }
 443
 444 static inline void inode_should_defrag(struct btrfs_inode *inode,
 445                 u64 start, u64 end, u64 num_bytes, u64 small_write)
 446 {
 447         /* If this is a small write inside eof, kick off a defrag */
 448         if (num_bytes < small_write &&
 449             (start > 0 || end + 1 < inode->disk_i_size))
 450                 btrfs_add_inode_defrag(NULL, inode);
 451 }
 452
 453 /*
 454  * we create compressed extents in two phases.  The first
 455  * phase compresses a range of pages that have already been
 456  * locked (both pages and state bits are locked).
 457  *
 458  * This is done inside an ordered work queue, and the compression
 459  * is spread across many cpus.  The actual IO submission is step
 460  * two, and the ordered work queue takes care of making sure that
 461  * happens in the same order things were put onto the queue by
 462  * writepages and friends.
 463  *
 464  * If this code finds it can't get good compression, it puts an
 465  * entry onto the work queue to write the uncompressed bytes.  This
 466  * makes sure that both compressed inodes and uncompressed inodes
 467  * are written in the same order that the flusher thread sent them
 468  * down.
 469  */
 470 static noinline int compress_file_range(struct async_chunk *async_chunk)
 471 {
 472         struct inode *inode = async_chunk->inode;
 473         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 474         u64 blocksize = fs_info->sectorsize;
 475         u64 start = async_chunk->start;
 476         u64 end = async_chunk->end;
 477         u64 actual_end;
 478         u64 i_size;
 479         int ret = 0;
 480         struct page **pages = NULL;
 481         unsigned long nr_pages;
 482         unsigned long total_compressed = 0;
 483         unsigned long total_in = 0;
 484         int i;
 485         int will_compress;
 486         int compress_type = fs_info->compress_type;
 487         int compressed_extents = 0;
 488         int redirty = 0;
 489
 490         inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1,
 491                         SZ_16K);
 492
 493         /*
 494          * We need to save i_size before now because it could change in between
 495          * us evaluating the size and assigning it.  This is because we lock and
 496          * unlock the page in truncate and fallocate, and then modify the i_size
 497          * later on.
 498          *
 499          * The barriers are to emulate READ_ONCE, remove that once i_size_read
 500          * does that for us.
 501          */
 502         barrier();
 503         i_size = i_size_read(inode);
 504         barrier();
 505         actual_end = min_t(u64, i_size, end + 1);
 506 again:
 507         will_compress = 0;
 508         nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
 509         BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0);
 510         nr_pages = min_t(unsigned long, nr_pages,
 511                         BTRFS_MAX_COMPRESSED / PAGE_SIZE);
 512
 513         /*
 514          * we don't want to send crud past the end of i_size through
 515          * compression, that's just a waste of CPU time.  So, if the
 516          * end of the file is before the start of our current
 517          * requested range of bytes, we bail out to the uncompressed
 518          * cleanup code that can deal with all of this.
 519          *
 520          * It isn't really the fastest way to fix things, but this is a
 521          * very uncommon corner.
 522          */
 523         if (actual_end <= start)
 524                 goto cleanup_and_bail_uncompressed;
 525
 526         total_compressed = actual_end - start;
 527
 528         /*
 529          * skip compression for a small file range(<=blocksize) that
 530          * isn't an inline extent, since it doesn't save disk space at all.
 531          */
 532         if (total_compressed <= blocksize &&
 533            (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
 534                 goto cleanup_and_bail_uncompressed;
 535
 536         total_compressed = min_t(unsigned long, total_compressed,
 537                         BTRFS_MAX_UNCOMPRESSED);
 538         total_in = 0;
 539         ret = 0;
 540
 541         /*
 542          * we do compression for mount -o compress and when the
 543          * inode has not been flagged as nocompress.  This flag can
 544          * change at any time if we discover bad compression ratios.
 545          */
 546         if (inode_need_compress(inode, start, end)) {
 547                 WARN_ON(pages);
 548                 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
 549                 if (!pages) {
 550                         /* just bail out to the uncompressed code */
 551                         nr_pages = 0;
 552                         goto cont;
 553                 }
 554
 555                 if (BTRFS_I(inode)->defrag_compress)
 556                         compress_type = BTRFS_I(inode)->defrag_compress;
 557                 else if (BTRFS_I(inode)->prop_compress)
 558                         compress_type = BTRFS_I(inode)->prop_compress;
 559
 560                 /*
 561                  * we need to call clear_page_dirty_for_io on each
 562                  * page in the range.  Otherwise applications with the file
 563                  * mmap'd can wander in and change the page contents while
 564                  * we are compressing them.
 565                  *
 566                  * If the compression fails for any reason, we set the pages
 567                  * dirty again later on.
 568                  *
 569                  * Note that the remaining part is redirtied, the start pointer
 570                  * has moved, the end is the original one.
 571                  */
 572                 if (!redirty) {
 573                         extent_range_clear_dirty_for_io(inode, start, end);
 574                         redirty = 1;
 575                 }
 576
 577                 /* Compression level is applied here and only here */
 578                 ret = btrfs_compress_pages(
 579                         compress_type | (fs_info->compress_level << 4),
 580                                            inode->i_mapping, start,
 581                                            pages,
 582                                            &nr_pages,
 583                                            &total_in,
 584                                            &total_compressed);
 585
 586                 if (!ret) {
 587                         unsigned long offset = offset_in_page(total_compressed);
 588                         struct page *page = pages[nr_pages - 1];
 589                         char *kaddr;
 590
 591                         /* zero the tail end of the last page, we might be
 592                          * sending it down to disk
 593                          */
 594                         if (offset) {
 595                                 kaddr = kmap_atomic(page);
 596                                 memset(kaddr + offset, 0,
 597                                        PAGE_SIZE - offset);
 598                                 kunmap_atomic(kaddr);
 599                         }
 600                         will_compress = 1;
 601                 }
 602         }
 603 cont:
 604         if (start == 0) {
 605                 /* lets try to make an inline extent */
 606                 if (ret || total_in < actual_end) {
 607                         /* we didn't compress the entire range, try
 608                          * to make an uncompressed inline extent.
 609                          */
 610                         ret = cow_file_range_inline(inode, start, end, 0,
 611                                                     BTRFS_COMPRESS_NONE, NULL);
 612                 } else {
 613                         /* try making a compressed inline extent */
 614                         ret = cow_file_range_inline(inode, start, end,
 615                                                     total_compressed,
 616                                                     compress_type, pages);
 617                 }
 618                 if (ret <= 0) {
 619                         unsigned long clear_flags = EXTENT_DELALLOC |
 620                                 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
 621                                 EXTENT_DO_ACCOUNTING;
 622                         unsigned long page_error_op;
 623
 624                         page_error_op = ret < 0 ? PAGE_SET_ERROR : 0;
 625
 626                         /*
 627                          * inline extent creation worked or returned error,
 628                          * we don't need to create any more async work items.
 629                          * Unlock and free up our temp pages.
 630                          *
 631                          * We use DO_ACCOUNTING here because we need the
 632                          * delalloc_release_metadata to be done _after_ we drop
 633                          * our outstanding extent for clearing delalloc for this
 634                          * range.
 635                          */
 636                         extent_clear_unlock_delalloc(inode, start, end, NULL,
 637                                                      clear_flags,
 638                                                      PAGE_UNLOCK |
 639                                                      PAGE_CLEAR_DIRTY |
 640                                                      PAGE_SET_WRITEBACK |
 641                                                      page_error_op |
 642                                                      PAGE_END_WRITEBACK);
 643
 644                         /*
 645                          * Ensure we only free the compressed pages if we have
 646                          * them allocated, as we can still reach here with
 647                          * inode_need_compress() == false.
 648                          */
 649                         if (pages) {
 650                                 for (i = 0; i < nr_pages; i++) {
 651                                         WARN_ON(pages[i]->mapping);
 652                                         put_page(pages[i]);
 653                                 }
 654                                 kfree(pages);
 655                         }
 656                         return 0;
 657                 }
 658         }
 659
 660         if (will_compress) {
 661                 /*
 662                  * we aren't doing an inline extent round the compressed size
 663                  * up to a block size boundary so the allocator does sane
 664                  * things
 665                  */
 666                 total_compressed = ALIGN(total_compressed, blocksize);
 667
 668                 /*
 669                  * one last check to make sure the compression is really a
 670                  * win, compare the page count read with the blocks on disk,
 671                  * compression must free at least one sector size
 672                  */
 673                 total_in = ALIGN(total_in, PAGE_SIZE);
 674                 if (total_compressed + blocksize <= total_in) {
 675                         compressed_extents++;
 676
 677                         /*
 678                          * The async work queues will take care of doing actual
 679                          * allocation on disk for these compressed pages, and
 680                          * will submit them to the elevator.
 681                          */
 682                         add_async_extent(async_chunk, start, total_in,
 683                                         total_compressed, pages, nr_pages,
 684                                         compress_type);
 685
 686                         if (start + total_in < end) {
 687                                 start += total_in;
 688                                 pages = NULL;
 689                                 cond_resched();
 690                                 goto again;
 691                         }
 692                         return compressed_extents;
 693                 }
 694         }
 695         if (pages) {
 696                 /*
 697                  * the compression code ran but failed to make things smaller,
 698                  * free any pages it allocated and our page pointer array
 699                  */
 700                 for (i = 0; i < nr_pages; i++) {
 701                         WARN_ON(pages[i]->mapping);
 702                         put_page(pages[i]);
 703                 }
 704                 kfree(pages);
 705                 pages = NULL;
 706                 total_compressed = 0;
 707                 nr_pages = 0;
 708
 709                 /* flag the file so we don't compress in the future */
 710                 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) &&
 711                     !(BTRFS_I(inode)->prop_compress)) {
 712                         BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
 713                 }
 714         }
 715 cleanup_and_bail_uncompressed:
 716         /*
 717          * No compression, but we still need to write the pages in the file
 718          * we've been given so far.  redirty the locked page if it corresponds
 719          * to our extent and set things up for the async work queue to run
 720          * cow_file_range to do the normal delalloc dance.
 721          */
 722         if (async_chunk->locked_page &&
 723             (page_offset(async_chunk->locked_page) >= start &&
 724              page_offset(async_chunk->locked_page)) <= end) {
 725                 __set_page_dirty_nobuffers(async_chunk->locked_page);
 726                 /* unlocked later on in the async handlers */
 727         }
 728
 729         if (redirty)
 730                 extent_range_redirty_for_io(inode, start, end);
 731         add_async_extent(async_chunk, start, end - start + 1, 0, NULL, 0,
 732                          BTRFS_COMPRESS_NONE);
 733         compressed_extents++;
 734
 735         return compressed_extents;
 736 }
 737
 738 static void free_async_extent_pages(struct async_extent *async_extent)
 739 {
 740         int i;
 741
 742         if (!async_extent->pages)
 743                 return;
 744
 745         for (i = 0; i < async_extent->nr_pages; i++) {
 746                 WARN_ON(async_extent->pages[i]->mapping);
 747                 put_page(async_extent->pages[i]);
 748         }
 749         kfree(async_extent->pages);
 750         async_extent->nr_pages = 0;
 751         async_extent->pages = NULL;
 752 }
 753
 754 /*
 755  * phase two of compressed writeback.  This is the ordered portion
 756  * of the code, which only gets called in the order the work was
 757  * queued.  We walk all the async extents created by compress_file_range
 758  * and send them down to the disk.
 759  */
 760 static noinline void submit_compressed_extents(struct async_chunk *async_chunk)
 761 {
 762         struct inode *inode = async_chunk->inode;
 763         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 764         struct async_extent *async_extent;
 765         u64 alloc_hint = 0;
 766         struct btrfs_key ins;
 767         struct extent_map *em;
 768         struct btrfs_root *root = BTRFS_I(inode)->root;
 769         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
 770         int ret = 0;
 771
 772 again:
 773         while (!list_empty(&async_chunk->extents)) {
 774                 async_extent = list_entry(async_chunk->extents.next,
 775                                           struct async_extent, list);
 776                 list_del(&async_extent->list);
 777
 778 retry:
 779                 lock_extent(io_tree, async_extent->start,
 780                             async_extent->start + async_extent->ram_size - 1);
 781                 /* did the compression code fall back to uncompressed IO? */
 782                 if (!async_extent->pages) {
 783                         int page_started = 0;
 784                         unsigned long nr_written = 0;
 785
 786                         /* allocate blocks */
 787                         ret = cow_file_range(inode, async_chunk->locked_page,
 788                                              async_extent->start,
 789                                              async_extent->start +
 790                                              async_extent->ram_size - 1,
 791                                              &page_started, &nr_written, 0);
 792
 793                         /* JDM XXX */
 794
 795                         /*
 796                          * if page_started, cow_file_range inserted an
 797                          * inline extent and took care of all the unlocking
 798                          * and IO for us.  Otherwise, we need to submit
 799                          * all those pages down to the drive.
 800                          */
 801                         if (!page_started && !ret)
 802                                 extent_write_locked_range(inode,
 803                                                   async_extent->start,
 804                                                   async_extent->start +
 805                                                   async_extent->ram_size - 1,
 806                                                   WB_SYNC_ALL);
 807                         else if (ret && async_chunk->locked_page)
 808                                 unlock_page(async_chunk->locked_page);
 809                         kfree(async_extent);
 810                         cond_resched();
 811                         continue;
 812                 }
 813
 814                 ret = btrfs_reserve_extent(root, async_extent->ram_size,
 815                                            async_extent->compressed_size,
 816                                            async_extent->compressed_size,
 817                                            0, alloc_hint, &ins, 1, 1);
 818                 if (ret) {
 819                         free_async_extent_pages(async_extent);
 820
 821                         if (ret == -ENOSPC) {
 822                                 unlock_extent(io_tree, async_extent->start,
 823                                               async_extent->start +
 824                                               async_extent->ram_size - 1);
 825
 826                                 /*
 827                                  * we need to redirty the pages if we decide to
 828                                  * fallback to uncompressed IO, otherwise we
 829                                  * will not submit these pages down to lower
 830                                  * layers.
 831                                  */
 832                                 extent_range_redirty_for_io(inode,
 833                                                 async_extent->start,
 834                                                 async_extent->start +
 835                                                 async_extent->ram_size - 1);
 836
 837                                 goto retry;
 838                         }
 839                         goto out_free;
 840                 }
 841                 /*
 842                  * here we're doing allocation and writeback of the
 843                  * compressed pages
 844                  */
 845                 em = create_io_em(inode, async_extent->start,
 846                                   async_extent->ram_size, /* len */
 847                                   async_extent->start, /* orig_start */
 848                                   ins.objectid, /* block_start */
 849                                   ins.offset, /* block_len */
 850                                   ins.offset, /* orig_block_len */
 851                                   async_extent->ram_size, /* ram_bytes */
 852                                   async_extent->compress_type,
 853                                   BTRFS_ORDERED_COMPRESSED);
 854                 if (IS_ERR(em))
 855                         /* ret value is not necessary due to void function */
 856                         goto out_free_reserve;
 857                 free_extent_map(em);
 858
 859                 ret = btrfs_add_ordered_extent_compress(inode,
 860                                                 async_extent->start,
 861                                                 ins.objectid,
 862                                                 async_extent->ram_size,
 863                                                 ins.offset,
 864                                                 BTRFS_ORDERED_COMPRESSED,
 865                                                 async_extent->compress_type);
 866                 if (ret) {
 867                         btrfs_drop_extent_cache(BTRFS_I(inode),
 868                                                 async_extent->start,
 869                                                 async_extent->start +
 870                                                 async_extent->ram_size - 1, 0);
 871                         goto out_free_reserve;
 872                 }
 873                 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
 874
 875                 /*
 876                  * clear dirty, set writeback and unlock the pages.
 877                  */
 878                 extent_clear_unlock_delalloc(inode, async_extent->start,
 879                                 async_extent->start +
 880                                 async_extent->ram_size - 1,
 881                                 NULL, EXTENT_LOCKED | EXTENT_DELALLOC,
 882                                 PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
 883                                 PAGE_SET_WRITEBACK);
 884                 if (btrfs_submit_compressed_write(inode,
 885                                     async_extent->start,
 886                                     async_extent->ram_size,
 887                                     ins.objectid,
 888                                     ins.offset, async_extent->pages,
 889                                     async_extent->nr_pages,
 890                                     async_chunk->write_flags)) {
 891                         struct page *p = async_extent->pages[0];
 892                         const u64 start = async_extent->start;
 893                         const u64 end = start + async_extent->ram_size - 1;
 894
 895                         p->mapping = inode->i_mapping;
 896                         btrfs_writepage_endio_finish_ordered(p, start, end, 0);
 897
 898                         p->mapping = NULL;
 899                         extent_clear_unlock_delalloc(inode, start, end,
 900                                                      NULL, 0,
 901                                                      PAGE_END_WRITEBACK |
 902                                                      PAGE_SET_ERROR);
 903                         free_async_extent_pages(async_extent);
 904                 }
 905                 alloc_hint = ins.objectid + ins.offset;
 906                 kfree(async_extent);
 907                 cond_resched();
 908         }
 909         return;
 910 out_free_reserve:
 911         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
 912         btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
 913 out_free:
 914         extent_clear_unlock_delalloc(inode, async_extent->start,
 915                                      async_extent->start +
 916                                      async_extent->ram_size - 1,
 917                                      NULL, EXTENT_LOCKED | EXTENT_DELALLOC |
 918                                      EXTENT_DELALLOC_NEW |
 919                                      EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
 920                                      PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
 921                                      PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
 922                                      PAGE_SET_ERROR);
 923         free_async_extent_pages(async_extent);
 924         kfree(async_extent);
 925         goto again;
 926 }
 927
 928 static u64 get_extent_allocation_hint(struct inode *inode, u64 start,
 929                                       u64 num_bytes)
 930 {
 931         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
 932         struct extent_map *em;
 933         u64 alloc_hint = 0;
 934
 935         read_lock(&em_tree->lock);
 936         em = search_extent_mapping(em_tree, start, num_bytes);
 937         if (em) {
 938                 /*
 939                  * if block start isn't an actual block number then find the
 940                  * first block in this inode and use that as a hint.  If that
 941                  * block is also bogus then just don't worry about it.
 942                  */
 943                 if (em->block_start >= EXTENT_MAP_LAST_BYTE) {
 944                         free_extent_map(em);
 945                         em = search_extent_mapping(em_tree, 0, 0);
 946                         if (em && em->block_start < EXTENT_MAP_LAST_BYTE)
 947                                 alloc_hint = em->block_start;
 948                         if (em)
 949                                 free_extent_map(em);
 950                 } else {
 951                         alloc_hint = em->block_start;
 952                         free_extent_map(em);
 953                 }
 954         }
 955         read_unlock(&em_tree->lock);
 956
 957         return alloc_hint;
 958 }
 959
 960 /*
 961  * when extent_io.c finds a delayed allocation range in the file,
 962  * the call backs end up in this code.  The basic idea is to
 963  * allocate extents on disk for the range, and create ordered data structs
 964  * in ram to track those extents.
 965  *
 966  * locked_page is the page that writepage had locked already.  We use
 967  * it to make sure we don't do extra locks or unlocks.
 968  *
 969  * *page_started is set to one if we unlock locked_page and do everything
 970  * required to start IO on it.  It may be clean and already done with
 971  * IO when we return.
 972  */
 973 static noinline int cow_file_range(struct inode *inode,
 974                                    struct page *locked_page,
 975                                    u64 start, u64 end, int *page_started,
 976                                    unsigned long *nr_written, int unlock)
 977 {
 978         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 979         struct btrfs_root *root = BTRFS_I(inode)->root;
 980         u64 alloc_hint = 0;
 981         u64 num_bytes;
 982         unsigned long ram_size;
 983         u64 cur_alloc_size = 0;
 984         u64 min_alloc_size;
 985         u64 blocksize = fs_info->sectorsize;
 986         struct btrfs_key ins;
 987         struct extent_map *em;
 988         unsigned clear_bits;
 989         unsigned long page_ops;
 990         bool extent_reserved = false;
 991         int ret = 0;
 992
 993         if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
 994                 WARN_ON_ONCE(1);
 995                 ret = -EINVAL;
 996                 goto out_unlock;
 997         }
 998
 999         num_bytes = ALIGN(end - start + 1, blocksize);
1000         num_bytes = max(blocksize,  num_bytes);
1001         ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy));
1002
1003         inode_should_defrag(BTRFS_I(inode), start, end, num_bytes, SZ_64K);
1004
1005         if (start == 0) {
1006                 /* lets try to make an inline extent */
1007                 ret = cow_file_range_inline(inode, start, end, 0,
1008                                             BTRFS_COMPRESS_NONE, NULL);
1009                 if (ret == 0) {
1010                         /*
1011                          * We use DO_ACCOUNTING here because we need the
1012                          * delalloc_release_metadata to be run _after_ we drop
1013                          * our outstanding extent for clearing delalloc for this
1014                          * range.
1015                          */
1016                         extent_clear_unlock_delalloc(inode, start, end, NULL,
1017                                      EXTENT_LOCKED | EXTENT_DELALLOC |
1018                                      EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
1019                                      EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
1020                                      PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
1021                                      PAGE_END_WRITEBACK);
1022                         *nr_written = *nr_written +
1023                              (end - start + PAGE_SIZE) / PAGE_SIZE;
1024                         *page_started = 1;
1025                         goto out;
1026                 } else if (ret < 0) {
1027                         goto out_unlock;
1028                 }
1029         }
1030
1031         alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
1032         btrfs_drop_extent_cache(BTRFS_I(inode), start,
1033                         start + num_bytes - 1, 0);
1034
1035         /*
1036          * Relocation relies on the relocated extents to have exactly the same
1037          * size as the original extents. Normally writeback for relocation data
1038          * extents follows a NOCOW path because relocation preallocates the
1039          * extents. However, due to an operation such as scrub turning a block
1040          * group to RO mode, it may fallback to COW mode, so we must make sure
1041          * an extent allocated during COW has exactly the requested size and can
1042          * not be split into smaller extents, otherwise relocation breaks and
1043          * fails during the stage where it updates the bytenr of file extent
1044          * items.
1045          */
1046         if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
1047                 min_alloc_size = num_bytes;
1048         else
1049                 min_alloc_size = fs_info->sectorsize;
1050
1051         while (num_bytes > 0) {
1052                 cur_alloc_size = num_bytes;
1053                 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size,
1054                                            min_alloc_size, 0, alloc_hint,
1055                                            &ins, 1, 1);
1056                 if (ret < 0)
1057                         goto out_unlock;
1058                 cur_alloc_size = ins.offset;
1059                 extent_reserved = true;
1060
1061                 ram_size = ins.offset;
1062                 em = create_io_em(inode, start, ins.offset, /* len */
1063                                   start, /* orig_start */
1064                                   ins.objectid, /* block_start */
1065                                   ins.offset, /* block_len */
1066                                   ins.offset, /* orig_block_len */
1067                                   ram_size, /* ram_bytes */
1068                                   BTRFS_COMPRESS_NONE, /* compress_type */
1069                                   BTRFS_ORDERED_REGULAR /* type */);
1070                 if (IS_ERR(em)) {
1071                         ret = PTR_ERR(em);
1072                         goto out_reserve;
1073                 }
1074                 free_extent_map(em);
1075
1076                 ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
1077                                                ram_size, cur_alloc_size, 0);
1078                 if (ret)
1079                         goto out_drop_extent_cache;
1080
1081                 if (root->root_key.objectid ==
1082                     BTRFS_DATA_RELOC_TREE_OBJECTID) {
1083                         ret = btrfs_reloc_clone_csums(inode, start,
1084                                                       cur_alloc_size);
1085                         /*
1086                          * Only drop cache here, and process as normal.
1087                          *
1088                          * We must not allow extent_clear_unlock_delalloc()
1089                          * at out_unlock label to free meta of this ordered
1090                          * extent, as its meta should be freed by
1091                          * btrfs_finish_ordered_io().
1092                          *
1093                          * So we must continue until @start is increased to
1094                          * skip current ordered extent.
1095                          */
1096                         if (ret)
1097                                 btrfs_drop_extent_cache(BTRFS_I(inode), start,
1098                                                 start + ram_size - 1, 0);
1099                 }
1100
1101                 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1102
1103                 /* we're not doing compressed IO, don't unlock the first
1104                  * page (which the caller expects to stay locked), don't
1105                  * clear any dirty bits and don't set any writeback bits
1106                  *
1107                  * Do set the Private2 bit so we know this page was properly
1108                  * setup for writepage
1109                  */
1110                 page_ops = unlock ? PAGE_UNLOCK : 0;
1111                 page_ops |= PAGE_SET_PRIVATE2;
1112
1113                 extent_clear_unlock_delalloc(inode, start,
1114                                              start + ram_size - 1,
1115                                              locked_page,
1116                                              EXTENT_LOCKED | EXTENT_DELALLOC,
1117                                              page_ops);
1118                 if (num_bytes < cur_alloc_size)
1119                         num_bytes = 0;
1120                 else
1121                         num_bytes -= cur_alloc_size;
1122                 alloc_hint = ins.objectid + ins.offset;
1123                 start += cur_alloc_size;
1124                 extent_reserved = false;
1125
1126                 /*
1127                  * btrfs_reloc_clone_csums() error, since start is increased
1128                  * extent_clear_unlock_delalloc() at out_unlock label won't
1129                  * free metadata of current ordered extent, we're OK to exit.
1130                  */
1131                 if (ret)
1132                         goto out_unlock;
1133         }
1134 out:
1135         return ret;
1136
1137 out_drop_extent_cache:
1138         btrfs_drop_extent_cache(BTRFS_I(inode), start, start + ram_size - 1, 0);
1139 out_reserve:
1140         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1141         btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
1142 out_unlock:
1143         clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
1144                 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV;
1145         page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
1146                 PAGE_END_WRITEBACK;
1147         /*
1148          * If we reserved an extent for our delalloc range (or a subrange) and
1149          * failed to create the respective ordered extent, then it means that
1150          * when we reserved the extent we decremented the extent's size from
1151          * the data space_info's bytes_may_use counter and incremented the
1152          * space_info's bytes_reserved counter by the same amount. We must make
1153          * sure extent_clear_unlock_delalloc() does not try to decrement again
1154          * the data space_info's bytes_may_use counter, therefore we do not pass
1155          * it the flag EXTENT_CLEAR_DATA_RESV.
1156          */
1157         if (extent_reserved) {
1158                 extent_clear_unlock_delalloc(inode, start,
1159                                              start + cur_alloc_size - 1,
1160                                              locked_page,
1161                                              clear_bits,
1162                                              page_ops);
1163                 start += cur_alloc_size;
1164                 if (start >= end)
1165                         goto out;
1166         }
1167         extent_clear_unlock_delalloc(inode, start, end, locked_page,
1168                                      clear_bits | EXTENT_CLEAR_DATA_RESV,
1169                                      page_ops);
1170         goto out;
1171 }
1172
1173 /*
1174  * work queue call back to started compression on a file and pages
1175  */
1176 static noinline void async_cow_start(struct btrfs_work *work)
1177 {
1178         struct async_chunk *async_chunk;
1179         int compressed_extents;
1180
1181         async_chunk = container_of(work, struct async_chunk, work);
1182
1183         compressed_extents = compress_file_range(async_chunk);
1184         if (compressed_extents == 0) {
1185                 btrfs_add_delayed_iput(async_chunk->inode);
1186                 async_chunk->inode = NULL;
1187         }
1188 }
1189
1190 /*
1191  * work queue call back to submit previously compressed pages
1192  */
1193 static noinline void async_cow_submit(struct btrfs_work *work)
1194 {
1195         struct async_chunk *async_chunk = container_of(work, struct async_chunk,
1196                                                      work);
1197         struct btrfs_fs_info *fs_info = btrfs_work_owner(work);
1198         unsigned long nr_pages;
1199
1200         nr_pages = (async_chunk->end - async_chunk->start + PAGE_SIZE) >>
1201                 PAGE_SHIFT;
1202
1203         /*
1204          * ->inode could be NULL if async_chunk_start has failed to compress,
1205          * in which case we don't have anything to submit, yet we need to
1206          * always adjust ->async_delalloc_pages as its paired with the init
1207          * happening in cow_file_range_async
1208          */
1209         if (async_chunk->inode)
1210                 submit_compressed_extents(async_chunk);
1211
1212         /* atomic_sub_return implies a barrier */
1213         if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) <
1214             5 * SZ_1M)
1215                 cond_wake_up_nomb(&fs_info->async_submit_wait);
1216 }
1217
1218 static noinline void async_cow_free(struct btrfs_work *work)
1219 {
1220         struct async_chunk *async_chunk;
1221
1222         async_chunk = container_of(work, struct async_chunk, work);
1223         if (async_chunk->inode)
1224                 btrfs_add_delayed_iput(async_chunk->inode);
1225         /*
1226          * Since the pointer to 'pending' is at the beginning of the array of
1227          * async_chunk's, freeing it ensures the whole array has been freed.
1228          */
1229         if (atomic_dec_and_test(async_chunk->pending))
1230                 kvfree(async_chunk->pending);
1231 }
1232
1233 static int cow_file_range_async(struct inode *inode, struct page *locked_page,
1234                                 u64 start, u64 end, int *page_started,
1235                                 unsigned long *nr_written,
1236                                 unsigned int write_flags)
1237 {
1238         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1239         struct async_cow *ctx;
1240         struct async_chunk *async_chunk;
1241         unsigned long nr_pages;
1242         u64 cur_end;
1243         u64 num_chunks = DIV_ROUND_UP(end - start, SZ_512K);
1244         int i;
1245         bool should_compress;
1246         unsigned nofs_flag;
1247
1248         unlock_extent(&BTRFS_I(inode)->io_tree, start, end);
1249
1250         if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS &&
1251             !btrfs_test_opt(fs_info, FORCE_COMPRESS)) {
1252                 num_chunks = 1;
1253                 should_compress = false;
1254         } else {
1255                 should_compress = true;
1256         }
1257
1258         nofs_flag = memalloc_nofs_save();
1259         ctx = kvmalloc(struct_size(ctx, chunks, num_chunks), GFP_KERNEL);
1260         memalloc_nofs_restore(nofs_flag);
1261
1262         if (!ctx) {
1263                 unsigned clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC |
1264                         EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
1265                         EXTENT_DO_ACCOUNTING;
1266                 unsigned long page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
1267                         PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
1268                         PAGE_SET_ERROR;
1269
1270                 extent_clear_unlock_delalloc(inode, start, end, locked_page,
1271                                              clear_bits, page_ops);
1272                 return -ENOMEM;
1273         }
1274
1275         async_chunk = ctx->chunks;
1276         atomic_set(&ctx->num_chunks, num_chunks);
1277
1278         for (i = 0; i < num_chunks; i++) {
1279                 if (should_compress)
1280                         cur_end = min(end, start + SZ_512K - 1);
1281                 else
1282                         cur_end = end;
1283
1284                 /*
1285                  * igrab is called higher up in the call chain, take only the
1286                  * lightweight reference for the callback lifetime
1287                  */
1288                 ihold(inode);
1289                 async_chunk[i].pending = &ctx->num_chunks;
1290                 async_chunk[i].inode = inode;
1291                 async_chunk[i].start = start;
1292                 async_chunk[i].end = cur_end;
1293                 async_chunk[i].write_flags = write_flags;
1294                 INIT_LIST_HEAD(&async_chunk[i].extents);
1295
1296                 /*
1297                  * The locked_page comes all the way from writepage and its
1298                  * the original page we were actually given.  As we spread
1299                  * this large delalloc region across multiple async_chunk
1300                  * structs, only the first struct needs a pointer to locked_page
1301                  *
1302                  * This way we don't need racey decisions about who is supposed
1303                  * to unlock it.
1304                  */
1305                 if (locked_page) {
1306                         async_chunk[i].locked_page = locked_page;
1307                         locked_page = NULL;
1308                 } else {
1309                         async_chunk[i].locked_page = NULL;
1310                 }
1311
1312                 btrfs_init_work(&async_chunk[i].work, async_cow_start,
1313                                 async_cow_submit, async_cow_free);
1314
1315                 nr_pages = DIV_ROUND_UP(cur_end - start, PAGE_SIZE);
1316                 atomic_add(nr_pages, &fs_info->async_delalloc_pages);
1317
1318                 btrfs_queue_work(fs_info->delalloc_workers, &async_chunk[i].work);
1319
1320                 *nr_written += nr_pages;
1321                 start = cur_end + 1;
1322         }
1323         *page_started = 1;
1324         return 0;
1325 }
1326
1327 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info,
1328                                         u64 bytenr, u64 num_bytes)
1329 {
1330         int ret;
1331         struct btrfs_ordered_sum *sums;
1332         LIST_HEAD(list);
1333
1334         ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr,
1335                                        bytenr + num_bytes - 1, &list, 0);
1336         if (ret == 0 && list_empty(&list))
1337                 return 0;
1338
1339         while (!list_empty(&list)) {
1340                 sums = list_entry(list.next, struct btrfs_ordered_sum, list);
1341                 list_del(&sums->list);
1342                 kfree(sums);
1343         }
1344         if (ret < 0)
1345                 return ret;
1346         return 1;
1347 }
1348
1349 static int fallback_to_cow(struct inode *inode, struct page *locked_page,
1350                            const u64 start, const u64 end,
1351                            int *page_started, unsigned long *nr_written)
1352 {
1353         const bool is_space_ino = btrfs_is_free_space_inode(BTRFS_I(inode));
1354         const bool is_reloc_ino = (BTRFS_I(inode)->root->root_key.objectid ==
1355                                    BTRFS_DATA_RELOC_TREE_OBJECTID);
1356         const u64 range_bytes = end + 1 - start;
1357         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
1358         u64 range_start = start;
1359         u64 count;
1360
1361         /*
1362          * If EXTENT_NORESERVE is set it means that when the buffered write was
1363          * made we had not enough available data space and therefore we did not
1364          * reserve data space for it, since we though we could do NOCOW for the
1365          * respective file range (either there is prealloc extent or the inode
1366          * has the NOCOW bit set).
1367          *
1368          * However when we need to fallback to COW mode (because for example the
1369          * block group for the corresponding extent was turned to RO mode by a
1370          * scrub or relocation) we need to do the following:
1371          *
1372          * 1) We increment the bytes_may_use counter of the data space info.
1373          *    If COW succeeds, it allocates a new data extent and after doing
1374          *    that it decrements the space info's bytes_may_use counter and
1375          *    increments its bytes_reserved counter by the same amount (we do
1376          *    this at btrfs_add_reserved_bytes()). So we need to increment the
1377          *    bytes_may_use counter to compensate (when space is reserved at
1378          *    buffered write time, the bytes_may_use counter is incremented);
1379          *
1380          * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so
1381          *    that if the COW path fails for any reason, it decrements (through
1382          *    extent_clear_unlock_delalloc()) the bytes_may_use counter of the
1383          *    data space info, which we incremented in the step above.
1384          *
1385          * If we need to fallback to cow and the inode corresponds to a free
1386          * space cache inode or an inode of the data relocation tree, we must
1387          * also increment bytes_may_use of the data space_info for the same
1388          * reason. Space caches and relocated data extents always get a prealloc
1389          * extent for them, however scrub or balance may have set the block
1390          * group that contains that extent to RO mode and therefore force COW
1391          * when starting writeback.
1392          */
1393         count = count_range_bits(io_tree, &range_start, end, range_bytes,
1394                                  EXTENT_NORESERVE, 0);
1395         if (count > 0 || is_space_ino || is_reloc_ino) {
1396                 u64 bytes = count;
1397                 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
1398                 struct btrfs_space_info *sinfo = fs_info->data_sinfo;
1399
1400                 if (is_space_ino || is_reloc_ino)
1401                         bytes = range_bytes;
1402
1403                 spin_lock(&sinfo->lock);
1404                 btrfs_space_info_update_bytes_may_use(fs_info, sinfo, bytes);
1405                 spin_unlock(&sinfo->lock);
1406
1407                 if (count > 0)
1408                         clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE,
1409                                          0, 0, NULL);
1410         }
1411
1412         return cow_file_range(inode, locked_page, start, end, page_started,
1413                               nr_written, 1);
1414 }
1415
1416 /*
1417  * when nowcow writeback call back.  This checks for snapshots or COW copies
1418  * of the extents that exist in the file, and COWs the file as required.
1419  *
1420  * If no cow copies or snapshots exist, we write directly to the existing
1421  * blocks on disk
1422  */
1423 static noinline int run_delalloc_nocow(struct inode *inode,
1424                                        struct page *locked_page,
1425                                        const u64 start, const u64 end,
1426                                        int *page_started, int force,
1427                                        unsigned long *nr_written)
1428 {
1429         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1430         struct btrfs_root *root = BTRFS_I(inode)->root;
1431         struct btrfs_path *path;
1432         u64 cow_start = (u64)-1;
1433         u64 cur_offset = start;
1434         int ret;
1435         bool check_prev = true;
1436         const bool freespace_inode = btrfs_is_free_space_inode(BTRFS_I(inode));
1437         u64 ino = btrfs_ino(BTRFS_I(inode));
1438         bool nocow = false;
1439         u64 disk_bytenr = 0;
1440
1441         path = btrfs_alloc_path();
1442         if (!path) {
1443                 extent_clear_unlock_delalloc(inode, start, end, locked_page,
1444                                              EXTENT_LOCKED | EXTENT_DELALLOC |
1445                                              EXTENT_DO_ACCOUNTING |
1446                                              EXTENT_DEFRAG, PAGE_UNLOCK |
1447                                              PAGE_CLEAR_DIRTY |
1448                                              PAGE_SET_WRITEBACK |
1449                                              PAGE_END_WRITEBACK);
1450                 return -ENOMEM;
1451         }
1452
1453         while (1) {
1454                 struct btrfs_key found_key;
1455                 struct btrfs_file_extent_item *fi;
1456                 struct extent_buffer *leaf;
1457                 u64 extent_end;
1458                 u64 extent_offset;
1459                 u64 num_bytes = 0;
1460                 u64 disk_num_bytes;
1461                 u64 ram_bytes;
1462                 int extent_type;
1463
1464                 nocow = false;
1465
1466                 ret = btrfs_lookup_file_extent(NULL, root, path, ino,
1467                                                cur_offset, 0);
1468                 if (ret < 0)
1469                         goto error;
1470
1471                 /*
1472                  * If there is no extent for our range when doing the initial
1473                  * search, then go back to the previous slot as it will be the
1474                  * one containing the search offset
1475                  */
1476                 if (ret > 0 && path->slots[0] > 0 && check_prev) {
1477                         leaf = path->nodes[0];
1478                         btrfs_item_key_to_cpu(leaf, &found_key,
1479                                               path->slots[0] - 1);
1480                         if (found_key.objectid == ino &&
1481                             found_key.type == BTRFS_EXTENT_DATA_KEY)
1482                                 path->slots[0]--;
1483                 }
1484                 check_prev = false;
1485 next_slot:
1486                 /* Go to next leaf if we have exhausted the current one */
1487                 leaf = path->nodes[0];
1488                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
1489                         ret = btrfs_next_leaf(root, path);
1490                         if (ret < 0) {
1491                                 if (cow_start != (u64)-1)
1492                                         cur_offset = cow_start;
1493                                 goto error;
1494                         }
1495                         if (ret > 0)
1496                                 break;
1497                         leaf = path->nodes[0];
1498                 }
1499
1500                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1501
1502                 /* Didn't find anything for our INO */
1503                 if (found_key.objectid > ino)
1504                         break;
1505                 /*
1506                  * Keep searching until we find an EXTENT_ITEM or there are no
1507                  * more extents for this inode
1508                  */
1509                 if (WARN_ON_ONCE(found_key.objectid < ino) ||
1510                     found_key.type < BTRFS_EXTENT_DATA_KEY) {
1511                         path->slots[0]++;
1512                         goto next_slot;
1513                 }
1514
1515                 /* Found key is not EXTENT_DATA_KEY or starts after req range */
1516                 if (found_key.type > BTRFS_EXTENT_DATA_KEY ||
1517                     found_key.offset > end)
1518                         break;
1519
1520                 /*
1521                  * If the found extent starts after requested offset, then
1522                  * adjust extent_end to be right before this extent begins
1523                  */
1524                 if (found_key.offset > cur_offset) {
1525                         extent_end = found_key.offset;
1526                         extent_type = 0;
1527                         goto out_check;
1528                 }
1529
1530                 /*
1531                  * Found extent which begins before our range and potentially
1532                  * intersect it
1533                  */
1534                 fi = btrfs_item_ptr(leaf, path->slots[0],
1535                                     struct btrfs_file_extent_item);
1536                 extent_type = btrfs_file_extent_type(leaf, fi);
1537
1538                 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
1539                 if (extent_type == BTRFS_FILE_EXTENT_REG ||
1540                     extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1541                         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1542                         extent_offset = btrfs_file_extent_offset(leaf, fi);
1543                         extent_end = found_key.offset +
1544                                 btrfs_file_extent_num_bytes(leaf, fi);
1545                         disk_num_bytes =
1546                                 btrfs_file_extent_disk_num_bytes(leaf, fi);
1547                         /*
1548                          * If the extent we got ends before our current offset,
1549                          * skip to the next extent.
1550                          */
1551                         if (extent_end <= cur_offset) {
1552                                 path->slots[0]++;
1553                                 goto next_slot;
1554                         }
1555                         /* Skip holes */
1556                         if (disk_bytenr == 0)
1557                                 goto out_check;
1558                         /* Skip compressed/encrypted/encoded extents */
1559                         if (btrfs_file_extent_compression(leaf, fi) ||
1560                             btrfs_file_extent_encryption(leaf, fi) ||
1561                             btrfs_file_extent_other_encoding(leaf, fi))
1562                                 goto out_check;
1563                         /*
1564                          * If extent is created before the last volume's snapshot
1565                          * this implies the extent is shared, hence we can't do
1566                          * nocow. This is the same check as in
1567                          * btrfs_cross_ref_exist but without calling
1568                          * btrfs_search_slot.
1569                          */
1570                         if (!freespace_inode &&
1571                             btrfs_file_extent_generation(leaf, fi) <=
1572                             btrfs_root_last_snapshot(&root->root_item))
1573                                 goto out_check;
1574                         if (extent_type == BTRFS_FILE_EXTENT_REG && !force)
1575                                 goto out_check;
1576                         /* If extent is RO, we must COW it */
1577                         if (btrfs_extent_readonly(fs_info, disk_bytenr))
1578                                 goto out_check;
1579                         ret = btrfs_cross_ref_exist(root, ino,
1580                                                     found_key.offset -
1581                                                     extent_offset, disk_bytenr, false);
1582                         if (ret) {
1583                                 /*
1584                                  * ret could be -EIO if the above fails to read
1585                                  * metadata.
1586                                  */
1587                                 if (ret < 0) {
1588                                         if (cow_start != (u64)-1)
1589                                                 cur_offset = cow_start;
1590                                         goto error;
1591                                 }
1592
1593                                 WARN_ON_ONCE(freespace_inode);
1594                                 goto out_check;
1595                         }
1596                         disk_bytenr += extent_offset;
1597                         disk_bytenr += cur_offset - found_key.offset;
1598                         num_bytes = min(end + 1, extent_end) - cur_offset;
1599                         /*
1600                          * If there are pending snapshots for this root, we
1601                          * fall into common COW way
1602                          */
1603                         if (!freespace_inode && atomic_read(&root->snapshot_force_cow))
1604                                 goto out_check;
1605                         /*
1606                          * force cow if csum exists in the range.
1607                          * this ensure that csum for a given extent are
1608                          * either valid or do not exist.
1609                          */
1610                         ret = csum_exist_in_range(fs_info, disk_bytenr,
1611                                                   num_bytes);
1612                         if (ret) {
1613                                 /*
1614                                  * ret could be -EIO if the above fails to read
1615                                  * metadata.
1616                                  */
1617                                 if (ret < 0) {
1618                                         if (cow_start != (u64)-1)
1619                                                 cur_offset = cow_start;
1620                                         goto error;
1621                                 }
1622                                 WARN_ON_ONCE(freespace_inode);
1623                                 goto out_check;
1624                         }
1625                         if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr))
1626                                 goto out_check;
1627                         nocow = true;
1628                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
1629                         extent_end = found_key.offset + ram_bytes;
1630                         extent_end = ALIGN(extent_end, fs_info->sectorsize);
1631                         /* Skip extents outside of our requested range */
1632                         if (extent_end <= start) {
1633                                 path->slots[0]++;
1634                                 goto next_slot;
1635                         }
1636                 } else {
1637                         /* If this triggers then we have a memory corruption */
1638                         BUG();
1639                 }
1640 out_check:
1641                 /*
1642                  * If nocow is false then record the beginning of the range
1643                  * that needs to be COWed
1644                  */
1645                 if (!nocow) {
1646                         if (cow_start == (u64)-1)
1647                                 cow_start = cur_offset;
1648                         cur_offset = extent_end;
1649                         if (cur_offset > end)
1650                                 break;
1651                         path->slots[0]++;
1652                         goto next_slot;
1653                 }
1654
1655                 btrfs_release_path(path);
1656
1657                 /*
1658                  * COW range from cow_start to found_key.offset - 1. As the key
1659                  * will contain the beginning of the first extent that can be
1660                  * NOCOW, following one which needs to be COW'ed
1661                  */
1662                 if (cow_start != (u64)-1) {
1663                         ret = fallback_to_cow(inode, locked_page, cow_start,
1664                                               found_key.offset - 1,
1665                                               page_started, nr_written);
1666                         if (ret)
1667                                 goto error;
1668                         cow_start = (u64)-1;
1669                 }
1670
1671                 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1672                         u64 orig_start = found_key.offset - extent_offset;
1673                         struct extent_map *em;
1674
1675                         em = create_io_em(inode, cur_offset, num_bytes,
1676                                           orig_start,
1677                                           disk_bytenr, /* block_start */
1678                                           num_bytes, /* block_len */
1679                                           disk_num_bytes, /* orig_block_len */
1680                                           ram_bytes, BTRFS_COMPRESS_NONE,
1681                                           BTRFS_ORDERED_PREALLOC);
1682                         if (IS_ERR(em)) {
1683                                 ret = PTR_ERR(em);
1684                                 goto error;
1685                         }
1686                         free_extent_map(em);
1687                         ret = btrfs_add_ordered_extent(inode, cur_offset,
1688                                                        disk_bytenr, num_bytes,
1689                                                        num_bytes,
1690                                                        BTRFS_ORDERED_PREALLOC);
1691                         if (ret) {
1692                                 btrfs_drop_extent_cache(BTRFS_I(inode),
1693                                                         cur_offset,
1694                                                         cur_offset + num_bytes - 1,
1695                                                         0);
1696                                 goto error;
1697                         }
1698                 } else {
1699                         ret = btrfs_add_ordered_extent(inode, cur_offset,
1700                                                        disk_bytenr, num_bytes,
1701                                                        num_bytes,
1702                                                        BTRFS_ORDERED_NOCOW);
1703                         if (ret)
1704                                 goto error;
1705                 }
1706
1707                 if (nocow)
1708                         btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1709                 nocow = false;
1710
1711                 if (root->root_key.objectid ==
1712                     BTRFS_DATA_RELOC_TREE_OBJECTID)
1713                         /*
1714                          * Error handled later, as we must prevent
1715                          * extent_clear_unlock_delalloc() in error handler
1716                          * from freeing metadata of created ordered extent.
1717                          */
1718                         ret = btrfs_reloc_clone_csums(inode, cur_offset,
1719                                                       num_bytes);
1720
1721                 extent_clear_unlock_delalloc(inode, cur_offset,
1722                                              cur_offset + num_bytes - 1,
1723                                              locked_page, EXTENT_LOCKED |
1724                                              EXTENT_DELALLOC |
1725                                              EXTENT_CLEAR_DATA_RESV,
1726                                              PAGE_UNLOCK | PAGE_SET_PRIVATE2);
1727
1728                 cur_offset = extent_end;
1729
1730                 /*
1731                  * btrfs_reloc_clone_csums() error, now we're OK to call error
1732                  * handler, as metadata for created ordered extent will only
1733                  * be freed by btrfs_finish_ordered_io().
1734                  */
1735                 if (ret)
1736                         goto error;
1737                 if (cur_offset > end)
1738                         break;
1739         }
1740         btrfs_release_path(path);
1741
1742         if (cur_offset <= end && cow_start == (u64)-1)
1743                 cow_start = cur_offset;
1744
1745         if (cow_start != (u64)-1) {
1746                 cur_offset = end;
1747                 ret = fallback_to_cow(inode, locked_page, cow_start, end,
1748                                       page_started, nr_written);
1749                 if (ret)
1750                         goto error;
1751         }
1752
1753 error:
1754         if (nocow)
1755                 btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1756
1757         if (ret && cur_offset < end)
1758                 extent_clear_unlock_delalloc(inode, cur_offset, end,
1759                                              locked_page, EXTENT_LOCKED |
1760                                              EXTENT_DELALLOC | EXTENT_DEFRAG |
1761                                              EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
1762                                              PAGE_CLEAR_DIRTY |
1763                                              PAGE_SET_WRITEBACK |
1764                                              PAGE_END_WRITEBACK);
1765         btrfs_free_path(path);
1766         return ret;
1767 }
1768
1769 static inline int need_force_cow(struct inode *inode, u64 start, u64 end)
1770 {
1771
1772         if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
1773             !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC))
1774                 return 0;
1775
1776         /*
1777          * @defrag_bytes is a hint value, no spinlock held here,
1778          * if is not zero, it means the file is defragging.
1779          * Force cow if given extent needs to be defragged.
1780          */
1781         if (BTRFS_I(inode)->defrag_bytes &&
1782             test_range_bit(&BTRFS_I(inode)->io_tree, start, end,
1783                            EXTENT_DEFRAG, 0, NULL))
1784                 return 1;
1785
1786         return 0;
1787 }
1788
1789 /*
1790  * Function to process delayed allocation (create CoW) for ranges which are
1791  * being touched for the first time.
1792  */
1793 int btrfs_run_delalloc_range(struct inode *inode, struct page *locked_page,
1794                 u64 start, u64 end, int *page_started, unsigned long *nr_written,
1795                 struct writeback_control *wbc)
1796 {
1797         int ret;
1798         int force_cow = need_force_cow(inode, start, end);
1799         unsigned int write_flags = wbc_to_write_flags(wbc);
1800
1801         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) {
1802                 ret = run_delalloc_nocow(inode, locked_page, start, end,
1803                                          page_started, 1, nr_written);
1804         } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) {
1805                 ret = run_delalloc_nocow(inode, locked_page, start, end,
1806                                          page_started, 0, nr_written);
1807         } else if (!inode_can_compress(inode) ||
1808                    !inode_need_compress(inode, start, end)) {
1809                 ret = cow_file_range(inode, locked_page, start, end,
1810                                       page_started, nr_written, 1);
1811         } else {
1812                 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
1813                         &BTRFS_I(inode)->runtime_flags);
1814                 ret = cow_file_range_async(inode, locked_page, start, end,
1815                                            page_started, nr_written,
1816                                            write_flags);
1817         }
1818         if (ret)
1819                 btrfs_cleanup_ordered_extents(inode, locked_page, start,
1820                                               end - start + 1);
1821         return ret;
1822 }
1823
1824 void btrfs_split_delalloc_extent(struct inode *inode,
1825                                  struct extent_state *orig, u64 split)
1826 {
1827         u64 size;
1828
1829         /* not delalloc, ignore it */
1830         if (!(orig->state & EXTENT_DELALLOC))
1831                 return;
1832
1833         size = orig->end - orig->start + 1;
1834         if (size > BTRFS_MAX_EXTENT_SIZE) {
1835                 u32 num_extents;
1836                 u64 new_size;
1837
1838                 /*
1839                  * See the explanation in btrfs_merge_delalloc_extent, the same
1840                  * applies here, just in reverse.
1841                  */
1842                 new_size = orig->end - split + 1;
1843                 num_extents = count_max_extents(new_size);
1844                 new_size = split - orig->start;
1845                 num_extents += count_max_extents(new_size);
1846                 if (count_max_extents(size) >= num_extents)
1847                         return;
1848         }
1849
1850         spin_lock(&BTRFS_I(inode)->lock);
1851         btrfs_mod_outstanding_extents(BTRFS_I(inode), 1);
1852         spin_unlock(&BTRFS_I(inode)->lock);
1853 }
1854
1855 /*
1856  * Handle merged delayed allocation extents so we can keep track of new extents
1857  * that are just merged onto old extents, such as when we are doing sequential
1858  * writes, so we can properly account for the metadata space we'll need.
1859  */
1860 void btrfs_merge_delalloc_extent(struct inode *inode, struct extent_state *new,
1861                                  struct extent_state *other)
1862 {
1863         u64 new_size, old_size;
1864         u32 num_extents;
1865
1866         /* not delalloc, ignore it */
1867         if (!(other->state & EXTENT_DELALLOC))
1868                 return;
1869
1870         if (new->start > other->start)
1871                 new_size = new->end - other->start + 1;
1872         else
1873                 new_size = other->end - new->start + 1;
1874
1875         /* we're not bigger than the max, unreserve the space and go */
1876         if (new_size <= BTRFS_MAX_EXTENT_SIZE) {
1877                 spin_lock(&BTRFS_I(inode)->lock);
1878                 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);
1879                 spin_unlock(&BTRFS_I(inode)->lock);
1880                 return;
1881         }
1882
1883         /*
1884          * We have to add up either side to figure out how many extents were
1885          * accounted for before we merged into one big extent.  If the number of
1886          * extents we accounted for is <= the amount we need for the new range
1887          * then we can return, otherwise drop.  Think of it like this
1888          *
1889          * [ 4k][MAX_SIZE]
1890          *
1891          * So we've grown the extent by a MAX_SIZE extent, this would mean we
1892          * need 2 outstanding extents, on one side we have 1 and the other side
1893          * we have 1 so they are == and we can return.  But in this case
1894          *
1895          * [MAX_SIZE+4k][MAX_SIZE+4k]
1896          *
1897          * Each range on their own accounts for 2 extents, but merged together
1898          * they are only 3 extents worth of accounting, so we need to drop in
1899          * this case.
1900          */
1901         old_size = other->end - other->start + 1;
1902         num_extents = count_max_extents(old_size);
1903         old_size = new->end - new->start + 1;
1904         num_extents += count_max_extents(old_size);
1905         if (count_max_extents(new_size) >= num_extents)
1906                 return;
1907
1908         spin_lock(&BTRFS_I(inode)->lock);
1909         btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);
1910         spin_unlock(&BTRFS_I(inode)->lock);
1911 }
1912
1913 static void btrfs_add_delalloc_inodes(struct btrfs_root *root,
1914                                       struct inode *inode)
1915 {
1916         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1917
1918         spin_lock(&root->delalloc_lock);
1919         if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
1920                 list_add_tail(&BTRFS_I(inode)->delalloc_inodes,
1921                               &root->delalloc_inodes);
1922                 set_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1923                         &BTRFS_I(inode)->runtime_flags);
1924                 root->nr_delalloc_inodes++;
1925                 if (root->nr_delalloc_inodes == 1) {
1926                         spin_lock(&fs_info->delalloc_root_lock);
1927                         BUG_ON(!list_empty(&root->delalloc_root));
1928                         list_add_tail(&root->delalloc_root,
1929                                       &fs_info->delalloc_roots);
1930                         spin_unlock(&fs_info->delalloc_root_lock);
1931                 }
1932         }
1933         spin_unlock(&root->delalloc_lock);
1934 }
1935
1936
1937 void __btrfs_del_delalloc_inode(struct btrfs_root *root,
1938                                 struct btrfs_inode *inode)
1939 {
1940         struct btrfs_fs_info *fs_info = root->fs_info;
1941
1942         if (!list_empty(&inode->delalloc_inodes)) {
1943                 list_del_init(&inode->delalloc_inodes);
1944                 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1945                           &inode->runtime_flags);
1946                 root->nr_delalloc_inodes--;
1947                 if (!root->nr_delalloc_inodes) {
1948                         ASSERT(list_empty(&root->delalloc_inodes));
1949                         spin_lock(&fs_info->delalloc_root_lock);
1950                         BUG_ON(list_empty(&root->delalloc_root));
1951                         list_del_init(&root->delalloc_root);
1952                         spin_unlock(&fs_info->delalloc_root_lock);
1953                 }
1954         }
1955 }
1956
1957 static void btrfs_del_delalloc_inode(struct btrfs_root *root,
1958                                      struct btrfs_inode *inode)
1959 {
1960         spin_lock(&root->delalloc_lock);
1961         __btrfs_del_delalloc_inode(root, inode);
1962         spin_unlock(&root->delalloc_lock);
1963 }
1964
1965 /*
1966  * Properly track delayed allocation bytes in the inode and to maintain the
1967  * list of inodes that have pending delalloc work to be done.
1968  */
1969 void btrfs_set_delalloc_extent(struct inode *inode, struct extent_state *state,
1970                                unsigned *bits)
1971 {
1972         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1973
1974         if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC))
1975                 WARN_ON(1);
1976         /*
1977          * set_bit and clear bit hooks normally require _irqsave/restore
1978          * but in this case, we are only testing for the DELALLOC
1979          * bit, which is only set or cleared with irqs on
1980          */
1981         if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1982                 struct btrfs_root *root = BTRFS_I(inode)->root;
1983                 u64 len = state->end + 1 - state->start;
1984                 u32 num_extents = count_max_extents(len);
1985                 bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode));
1986
1987                 spin_lock(&BTRFS_I(inode)->lock);
1988                 btrfs_mod_outstanding_extents(BTRFS_I(inode), num_extents);
1989                 spin_unlock(&BTRFS_I(inode)->lock);
1990
1991                 /* For sanity tests */
1992                 if (btrfs_is_testing(fs_info))
1993                         return;
1994
1995                 percpu_counter_add_batch(&fs_info->delalloc_bytes, len,
1996                                          fs_info->delalloc_batch);
1997                 spin_lock(&BTRFS_I(inode)->lock);
1998                 BTRFS_I(inode)->delalloc_bytes += len;
1999                 if (*bits & EXTENT_DEFRAG)
2000                         BTRFS_I(inode)->defrag_bytes += len;
2001                 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
2002                                          &BTRFS_I(inode)->runtime_flags))
2003                         btrfs_add_delalloc_inodes(root, inode);
2004                 spin_unlock(&BTRFS_I(inode)->lock);
2005         }
2006
2007         if (!(state->state & EXTENT_DELALLOC_NEW) &&
2008             (*bits & EXTENT_DELALLOC_NEW)) {
2009                 spin_lock(&BTRFS_I(inode)->lock);
2010                 BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 -
2011                         state->start;
2012                 spin_unlock(&BTRFS_I(inode)->lock);
2013         }
2014 }
2015
2016 /*
2017  * Once a range is no longer delalloc this function ensures that proper
2018  * accounting happens.
2019  */
2020 void btrfs_clear_delalloc_extent(struct inode *vfs_inode,
2021                                  struct extent_state *state, unsigned *bits)
2022 {
2023         struct btrfs_inode *inode = BTRFS_I(vfs_inode);
2024         struct btrfs_fs_info *fs_info = btrfs_sb(vfs_inode->i_sb);
2025         u64 len = state->end + 1 - state->start;
2026         u32 num_extents = count_max_extents(len);
2027
2028         if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) {
2029                 spin_lock(&inode->lock);
2030                 inode->defrag_bytes -= len;
2031                 spin_unlock(&inode->lock);
2032         }
2033
2034         /*
2035          * set_bit and clear bit hooks normally require _irqsave/restore
2036          * but in this case, we are only testing for the DELALLOC
2037          * bit, which is only set or cleared with irqs on
2038          */
2039         if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
2040                 struct btrfs_root *root = inode->root;
2041                 bool do_list = !btrfs_is_free_space_inode(inode);
2042
2043                 spin_lock(&inode->lock);
2044                 btrfs_mod_outstanding_extents(inode, -num_extents);
2045                 spin_unlock(&inode->lock);
2046
2047                 /*
2048                  * We don't reserve metadata space for space cache inodes so we
2049                  * don't need to call delalloc_release_metadata if there is an
2050                  * error.
2051                  */
2052                 if (*bits & EXTENT_CLEAR_META_RESV &&
2053                     root != fs_info->tree_root)
2054                         btrfs_delalloc_release_metadata(inode, len, false);
2055
2056                 /* For sanity tests. */
2057                 if (btrfs_is_testing(fs_info))
2058                         return;
2059
2060                 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID &&
2061                     do_list && !(state->state & EXTENT_NORESERVE) &&
2062                     (*bits & EXTENT_CLEAR_DATA_RESV))
2063                         btrfs_free_reserved_data_space_noquota(
2064                                         &inode->vfs_inode,
2065                                         state->start, len);
2066
2067                 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len,
2068                                          fs_info->delalloc_batch);
2069                 spin_lock(&inode->lock);
2070                 inode->delalloc_bytes -= len;
2071                 if (do_list && inode->delalloc_bytes == 0 &&
2072                     test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
2073                                         &inode->runtime_flags))
2074                         btrfs_del_delalloc_inode(root, inode);
2075                 spin_unlock(&inode->lock);
2076         }
2077
2078         if ((state->state & EXTENT_DELALLOC_NEW) &&
2079             (*bits & EXTENT_DELALLOC_NEW)) {
2080                 spin_lock(&inode->lock);
2081                 ASSERT(inode->new_delalloc_bytes >= len);
2082                 inode->new_delalloc_bytes -= len;
2083                 spin_unlock(&inode->lock);
2084         }
2085 }
2086
2087 /*
2088  * btrfs_bio_fits_in_stripe - Checks whether the size of the given bio will fit
2089  * in a chunk's stripe. This function ensures that bios do not span a
2090  * stripe/chunk
2091  *
2092  * @page - The page we are about to add to the bio
2093  * @size - size we want to add to the bio
2094  * @bio - bio we want to ensure is smaller than a stripe
2095  * @bio_flags - flags of the bio
2096  *
2097  * return 1 if page cannot be added to the bio
2098  * return 0 if page can be added to the bio
2099  * return error otherwise
2100  */
2101 int btrfs_bio_fits_in_stripe(struct page *page, size_t size, struct bio *bio,
2102                              unsigned long bio_flags)
2103 {
2104         struct inode *inode = page->mapping->host;
2105         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2106         u64 logical = (u64)bio->bi_iter.bi_sector << 9;
2107         u64 length = 0;
2108         u64 map_length;
2109         int ret;
2110         struct btrfs_io_geometry geom;
2111
2112         if (bio_flags & EXTENT_BIO_COMPRESSED)
2113                 return 0;
2114
2115         length = bio->bi_iter.bi_size;
2116         map_length = length;
2117         ret = btrfs_get_io_geometry(fs_info, btrfs_op(bio), logical, map_length,
2118                                     &geom);
2119         if (ret < 0)
2120                 return ret;
2121
2122         if (geom.len < length + size)
2123                 return 1;
2124         return 0;
2125 }
2126
2127 /*
2128  * in order to insert checksums into the metadata in large chunks,
2129  * we wait until bio submission time.   All the pages in the bio are
2130  * checksummed and sums are attached onto the ordered extent record.
2131  *
2132  * At IO completion time the cums attached on the ordered extent record
2133  * are inserted into the btree
2134  */
2135 static blk_status_t btrfs_submit_bio_start(void *private_data, struct bio *bio,
2136                                     u64 bio_offset)
2137 {
2138         struct inode *inode = private_data;
2139         blk_status_t ret = 0;
2140
2141         ret = btrfs_csum_one_bio(inode, bio, 0, 0);
2142         BUG_ON(ret); /* -ENOMEM */
2143         return 0;
2144 }
2145
2146 /*
2147  * extent_io.c submission hook. This does the right thing for csum calculation
2148  * on write, or reading the csums from the tree before a read.
2149  *
2150  * Rules about async/sync submit,
2151  * a) read:                             sync submit
2152  *
2153  * b) write without checksum:           sync submit
2154  *
2155  * c) write with checksum:
2156  *    c-1) if bio is issued by fsync:   sync submit
2157  *         (sync_writers != 0)
2158  *
2159  *    c-2) if root is reloc root:       sync submit
2160  *         (only in case of buffered IO)
2161  *
2162  *    c-3) otherwise:                   async submit
2163  */
2164 static blk_status_t btrfs_submit_bio_hook(struct inode *inode, struct bio *bio,
2165                                           int mirror_num,
2166                                           unsigned long bio_flags)
2167
2168 {
2169         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2170         struct btrfs_root *root = BTRFS_I(inode)->root;
2171         enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA;
2172         blk_status_t ret = 0;
2173         int skip_sum;
2174         int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
2175
2176         skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
2177
2178         if (btrfs_is_free_space_inode(BTRFS_I(inode)))
2179                 metadata = BTRFS_WQ_ENDIO_FREE_SPACE;
2180
2181         if (bio_op(bio) != REQ_OP_WRITE) {
2182                 ret = btrfs_bio_wq_end_io(fs_info, bio, metadata);
2183                 if (ret)
2184                         goto out;
2185
2186                 if (bio_flags & EXTENT_BIO_COMPRESSED) {
2187                         ret = btrfs_submit_compressed_read(inode, bio,
2188                                                            mirror_num,
2189                                                            bio_flags);
2190                         goto out;
2191                 } else if (!skip_sum) {
2192                         ret = btrfs_lookup_bio_sums(inode, bio, NULL);
2193                         if (ret)
2194                                 goto out;
2195                 }
2196                 goto mapit;
2197         } else if (async && !skip_sum) {
2198                 /* csum items have already been cloned */
2199                 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
2200                         goto mapit;
2201                 /* we're doing a write, do the async checksumming */
2202                 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, bio_flags,
2203                                           0, inode, btrfs_submit_bio_start);
2204                 goto out;
2205         } else if (!skip_sum) {
2206                 ret = btrfs_csum_one_bio(inode, bio, 0, 0);
2207                 if (ret)
2208                         goto out;
2209         }
2210
2211 mapit:
2212         ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
2213
2214 out:
2215         if (ret) {
2216                 bio->bi_status = ret;
2217                 bio_endio(bio);
2218         }
2219         return ret;
2220 }
2221
2222 /*
2223  * given a list of ordered sums record them in the inode.  This happens
2224  * at IO completion time based on sums calculated at bio submission time.
2225  */
2226 static noinline int add_pending_csums(struct btrfs_trans_handle *trans,
2227                              struct inode *inode, struct list_head *list)
2228 {
2229         struct btrfs_ordered_sum *sum;
2230         int ret;
2231
2232         list_for_each_entry(sum, list, list) {
2233                 trans->adding_csums = true;
2234                 ret = btrfs_csum_file_blocks(trans,
2235                        BTRFS_I(inode)->root->fs_info->csum_root, sum);
2236                 trans->adding_csums = false;
2237                 if (ret)
2238                         return ret;
2239         }
2240         return 0;
2241 }
2242
2243 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end,
2244                               unsigned int extra_bits,
2245                               struct extent_state **cached_state)
2246 {
2247         WARN_ON(PAGE_ALIGNED(end));
2248         return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end,
2249                                    extra_bits, cached_state);
2250 }
2251
2252 /* see btrfs_writepage_start_hook for details on why this is required */
2253 struct btrfs_writepage_fixup {
2254         struct page *page;
2255         struct inode *inode;
2256         struct btrfs_work work;
2257 };
2258
2259 static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
2260 {
2261         struct btrfs_writepage_fixup *fixup;
2262         struct btrfs_ordered_extent *ordered;
2263         struct extent_state *cached_state = NULL;
2264         struct extent_changeset *data_reserved = NULL;
2265         struct page *page;
2266         struct inode *inode;
2267         u64 page_start;
2268         u64 page_end;
2269         int ret = 0;
2270         bool free_delalloc_space = true;
2271
2272         fixup = container_of(work, struct btrfs_writepage_fixup, work);
2273         page = fixup->page;
2274         inode = fixup->inode;
2275         page_start = page_offset(page);
2276         page_end = page_offset(page) + PAGE_SIZE - 1;
2277
2278         /*
2279          * This is similar to page_mkwrite, we need to reserve the space before
2280          * we take the page lock.
2281          */
2282         ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
2283                                            PAGE_SIZE);
2284 again:
2285         lock_page(page);
2286
2287         /*
2288          * Before we queued this fixup, we took a reference on the page.
2289          * page->mapping may go NULL, but it shouldn't be moved to a different
2290          * address space.
2291          */
2292         if (!page->mapping || !PageDirty(page) || !PageChecked(page)) {
2293                 /*
2294                  * Unfortunately this is a little tricky, either
2295                  *
2296                  * 1) We got here and our page had already been dealt with and
2297                  *    we reserved our space, thus ret == 0, so we need to just
2298                  *    drop our space reservation and bail.  This can happen the
2299                  *    first time we come into the fixup worker, or could happen
2300                  *    while waiting for the ordered extent.
2301                  * 2) Our page was already dealt with, but we happened to get an
2302                  *    ENOSPC above from the btrfs_delalloc_reserve_space.  In
2303                  *    this case we obviously don't have anything to release, but
2304                  *    because the page was already dealt with we don't want to
2305                  *    mark the page with an error, so make sure we're resetting
2306                  *    ret to 0.  This is why we have this check _before_ the ret
2307                  *    check, because we do not want to have a surprise ENOSPC
2308                  *    when the page was already properly dealt with.
2309                  */
2310                 if (!ret) {
2311                         btrfs_delalloc_release_extents(BTRFS_I(inode),
2312                                                        PAGE_SIZE);
2313                         btrfs_delalloc_release_space(inode, data_reserved,
2314                                                      page_start, PAGE_SIZE,
2315                                                      true);
2316                 }
2317                 ret = 0;
2318                 goto out_page;
2319         }
2320
2321         /*
2322          * We can't mess with the page state unless it is locked, so now that
2323          * it is locked bail if we failed to make our space reservation.
2324          */
2325         if (ret)
2326                 goto out_page;
2327
2328         lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end,
2329                          &cached_state);
2330
2331         /* already ordered? We're done */
2332         if (PagePrivate2(page))
2333                 goto out_reserved;
2334
2335         ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
2336                                         PAGE_SIZE);
2337         if (ordered) {
2338                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start,
2339                                      page_end, &cached_state);
2340                 unlock_page(page);
2341                 btrfs_start_ordered_extent(inode, ordered, 1);
2342                 btrfs_put_ordered_extent(ordered);
2343                 goto again;
2344         }
2345
2346         ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0,
2347                                         &cached_state);
2348         if (ret)
2349                 goto out_reserved;
2350
2351         /*
2352          * Everything went as planned, we're now the owner of a dirty page with
2353          * delayed allocation bits set and space reserved for our COW
2354          * destination.
2355          *
2356          * The page was dirty when we started, nothing should have cleaned it.
2357          */
2358         BUG_ON(!PageDirty(page));
2359         free_delalloc_space = false;
2360 out_reserved:
2361         btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE);
2362         if (free_delalloc_space)
2363                 btrfs_delalloc_release_space(inode, data_reserved, page_start,
2364                                              PAGE_SIZE, true);
2365         unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end,
2366                              &cached_state);
2367 out_page:
2368         if (ret) {
2369                 /*
2370                  * We hit ENOSPC or other errors.  Update the mapping and page
2371                  * to reflect the errors and clean the page.
2372                  */
2373                 mapping_set_error(page->mapping, ret);
2374                 end_extent_writepage(page, ret, page_start, page_end);
2375                 clear_page_dirty_for_io(page);
2376                 SetPageError(page);
2377         }
2378         ClearPageChecked(page);
2379         unlock_page(page);
2380         put_page(page);
2381         kfree(fixup);
2382         extent_changeset_free(data_reserved);
2383         /*
2384          * As a precaution, do a delayed iput in case it would be the last iput
2385          * that could need flushing space. Recursing back to fixup worker would
2386          * deadlock.
2387          */
2388         btrfs_add_delayed_iput(inode);
2389 }
2390
2391 /*
2392  * There are a few paths in the higher layers of the kernel that directly
2393  * set the page dirty bit without asking the filesystem if it is a
2394  * good idea.  This causes problems because we want to make sure COW
2395  * properly happens and the data=ordered rules are followed.
2396  *
2397  * In our case any range that doesn't have the ORDERED bit set
2398  * hasn't been properly setup for IO.  We kick off an async process
2399  * to fix it up.  The async helper will wait for ordered extents, set
2400  * the delalloc bit and make it safe to write the page.
2401  */
2402 int btrfs_writepage_cow_fixup(struct page *page, u64 start, u64 end)
2403 {
2404         struct inode *inode = page->mapping->host;
2405         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2406         struct btrfs_writepage_fixup *fixup;
2407
2408         /* this page is properly in the ordered list */
2409         if (TestClearPagePrivate2(page))
2410                 return 0;
2411
2412         /*
2413          * PageChecked is set below when we create a fixup worker for this page,
2414          * don't try to create another one if we're already PageChecked()
2415          *
2416          * The extent_io writepage code will redirty the page if we send back
2417          * EAGAIN.
2418          */
2419         if (PageChecked(page))
2420                 return -EAGAIN;
2421
2422         fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
2423         if (!fixup)
2424                 return -EAGAIN;
2425
2426         /*
2427          * We are already holding a reference to this inode from
2428          * write_cache_pages.  We need to hold it because the space reservation
2429          * takes place outside of the page lock, and we can't trust
2430          * page->mapping outside of the page lock.
2431          */
2432         ihold(inode);
2433         SetPageChecked(page);
2434         get_page(page);
2435         btrfs_init_work(&fixup->work, btrfs_writepage_fixup_worker, NULL, NULL);
2436         fixup->page = page;
2437         fixup->inode = inode;
2438         btrfs_queue_work(fs_info->fixup_workers, &fixup->work);
2439
2440         return -EAGAIN;
2441 }
2442
2443 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
2444                                        struct inode *inode, u64 file_pos,
2445                                        u64 disk_bytenr, u64 disk_num_bytes,
2446                                        u64 num_bytes, u64 ram_bytes,
2447                                        u8 compression, u8 encryption,
2448                                        u16 other_encoding, int extent_type)
2449 {
2450         struct btrfs_root *root = BTRFS_I(inode)->root;
2451         struct btrfs_file_extent_item *fi;
2452         struct btrfs_path *path;
2453         struct extent_buffer *leaf;
2454         struct btrfs_key ins;
2455         u64 qg_released;
2456         int extent_inserted = 0;
2457         int ret;
2458
2459         path = btrfs_alloc_path();
2460         if (!path)
2461                 return -ENOMEM;
2462
2463         /*
2464          * we may be replacing one extent in the tree with another.
2465          * The new extent is pinned in the extent map, and we don't want
2466          * to drop it from the cache until it is completely in the btree.
2467          *
2468          * So, tell btrfs_drop_extents to leave this extent in the cache.
2469          * the caller is expected to unpin it and allow it to be merged
2470          * with the others.
2471          */
2472         ret = __btrfs_drop_extents(trans, root, inode, path, file_pos,
2473                                    file_pos + num_bytes, NULL, 0,
2474                                    1, sizeof(*fi), &extent_inserted);
2475         if (ret)
2476                 goto out;
2477
2478         if (!extent_inserted) {
2479                 ins.objectid = btrfs_ino(BTRFS_I(inode));
2480                 ins.offset = file_pos;
2481                 ins.type = BTRFS_EXTENT_DATA_KEY;
2482
2483                 path->leave_spinning = 1;
2484                 ret = btrfs_insert_empty_item(trans, root, path, &ins,
2485                                               sizeof(*fi));
2486                 if (ret)
2487                         goto out;
2488         }
2489         leaf = path->nodes[0];
2490         fi = btrfs_item_ptr(leaf, path->slots[0],
2491                             struct btrfs_file_extent_item);
2492         btrfs_set_file_extent_generation(leaf, fi, trans->transid);
2493         btrfs_set_file_extent_type(leaf, fi, extent_type);
2494         btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr);
2495         btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes);
2496         btrfs_set_file_extent_offset(leaf, fi, 0);
2497         btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2498         btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
2499         btrfs_set_file_extent_compression(leaf, fi, compression);
2500         btrfs_set_file_extent_encryption(leaf, fi, encryption);
2501         btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding);
2502
2503         btrfs_mark_buffer_dirty(leaf);
2504         btrfs_release_path(path);
2505
2506         inode_add_bytes(inode, num_bytes);
2507
2508         ins.objectid = disk_bytenr;
2509         ins.offset = disk_num_bytes;
2510         ins.type = BTRFS_EXTENT_ITEM_KEY;
2511
2512         /*
2513          * Release the reserved range from inode dirty range map, as it is
2514          * already moved into delayed_ref_head
2515          */
2516         ret = btrfs_qgroup_release_data(inode, file_pos, ram_bytes);
2517         if (ret < 0)
2518                 goto out;
2519         qg_released = ret;
2520         ret = btrfs_alloc_reserved_file_extent(trans, root,
2521                                                btrfs_ino(BTRFS_I(inode)),
2522                                                file_pos, qg_released, &ins);
2523 out:
2524         btrfs_free_path(path);
2525
2526         return ret;
2527 }
2528
2529 /* snapshot-aware defrag */
2530 struct sa_defrag_extent_backref {
2531         struct rb_node node;
2532         struct old_sa_defrag_extent *old;
2533         u64 root_id;
2534         u64 inum;
2535         u64 file_pos;
2536         u64 extent_offset;
2537         u64 num_bytes;
2538         u64 generation;
2539 };
2540
2541 struct old_sa_defrag_extent {
2542         struct list_head list;
2543         struct new_sa_defrag_extent *new;
2544
2545         u64 extent_offset;
2546         u64 bytenr;
2547         u64 offset;
2548         u64 len;
2549         int count;
2550 };
2551
2552 struct new_sa_defrag_extent {
2553         struct rb_root root;
2554         struct list_head head;
2555         struct btrfs_path *path;
2556         struct inode *inode;
2557         u64 file_pos;
2558         u64 len;
2559         u64 bytenr;
2560         u64 disk_len;
2561         u8 compress_type;
2562 };
2563
2564 static int backref_comp(struct sa_defrag_extent_backref *b1,
2565                         struct sa_defrag_extent_backref *b2)
2566 {
2567         if (b1->root_id < b2->root_id)
2568                 return -1;
2569         else if (b1->root_id > b2->root_id)
2570                 return 1;
2571
2572         if (b1->inum < b2->inum)
2573                 return -1;
2574         else if (b1->inum > b2->inum)
2575                 return 1;
2576
2577         if (b1->file_pos < b2->file_pos)
2578                 return -1;
2579         else if (b1->file_pos > b2->file_pos)
2580                 return 1;
2581
2582         /*
2583          * [------------------------------] ===> (a range of space)
2584          *     |<--->|   |<---->| =============> (fs/file tree A)
2585          * |<---------------------------->| ===> (fs/file tree B)
2586          *
2587          * A range of space can refer to two file extents in one tree while
2588          * refer to only one file extent in another tree.
2589          *
2590          * So we may process a disk offset more than one time(two extents in A)
2591          * and locate at the same extent(one extent in B), then insert two same
2592          * backrefs(both refer to the extent in B).
2593          */
2594         return 0;
2595 }
2596
2597 static void backref_insert(struct rb_root *root,
2598                            struct sa_defrag_extent_backref *backref)
2599 {
2600         struct rb_node **p = &root->rb_node;
2601         struct rb_node *parent = NULL;
2602         struct sa_defrag_extent_backref *entry;
2603         int ret;
2604
2605         while (*p) {
2606                 parent = *p;
2607                 entry = rb_entry(parent, struct sa_defrag_extent_backref, node);
2608
2609                 ret = backref_comp(backref, entry);
2610                 if (ret < 0)
2611                         p = &(*p)->rb_left;
2612                 else
2613                         p = &(*p)->rb_right;
2614         }
2615
2616         rb_link_node(&backref->node, parent, p);
2617         rb_insert_color(&backref->node, root);
2618 }
2619
2620 /*
2621  * Note the backref might has changed, and in this case we just return 0.
2622  */
2623 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id,
2624                                        void *ctx)
2625 {
2626         struct btrfs_file_extent_item *extent;
2627         struct old_sa_defrag_extent *old = ctx;
2628         struct new_sa_defrag_extent *new = old->new;
2629         struct btrfs_path *path = new->path;
2630         struct btrfs_key key;
2631         struct btrfs_root *root;
2632         struct sa_defrag_extent_backref *backref;
2633         struct extent_buffer *leaf;
2634         struct inode *inode = new->inode;
2635         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2636         int slot;
2637         int ret;
2638         u64 extent_offset;
2639         u64 num_bytes;
2640
2641         if (BTRFS_I(inode)->root->root_key.objectid == root_id &&
2642             inum == btrfs_ino(BTRFS_I(inode)))
2643                 return 0;
2644
2645         key.objectid = root_id;
2646         key.type = BTRFS_ROOT_ITEM_KEY;
2647         key.offset = (u64)-1;
2648
2649         root = btrfs_read_fs_root_no_name(fs_info, &key);
2650         if (IS_ERR(root)) {
2651                 if (PTR_ERR(root) == -ENOENT)
2652                         return 0;
2653                 WARN_ON(1);
2654                 btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu",
2655                          inum, offset, root_id);
2656                 return PTR_ERR(root);
2657         }
2658
2659         key.objectid = inum;
2660         key.type = BTRFS_EXTENT_DATA_KEY;
2661         if (offset > (u64)-1 << 32)
2662                 key.offset = 0;
2663         else
2664                 key.offset = offset;
2665
2666         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2667         if (WARN_ON(ret < 0))
2668                 return ret;
2669         ret = 0;
2670
2671         while (1) {
2672                 cond_resched();
2673
2674                 leaf = path->nodes[0];
2675                 slot = path->slots[0];
2676
2677                 if (slot >= btrfs_header_nritems(leaf)) {
2678                         ret = btrfs_next_leaf(root, path);
2679                         if (ret < 0) {
2680                                 goto out;
2681                         } else if (ret > 0) {
2682                                 ret = 0;
2683                                 goto out;
2684                         }
2685                         continue;
2686                 }
2687
2688                 path->slots[0]++;
2689
2690                 btrfs_item_key_to_cpu(leaf, &key, slot);
2691
2692                 if (key.objectid > inum)
2693                         goto out;
2694
2695                 if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY)
2696                         continue;
2697
2698                 extent = btrfs_item_ptr(leaf, slot,
2699                                         struct btrfs_file_extent_item);
2700
2701                 if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr)
2702                         continue;
2703
2704                 /*
2705                  * 'offset' refers to the exact key.offset,
2706                  * NOT the 'offset' field in btrfs_extent_data_ref, ie.
2707                  * (key.offset - extent_offset).
2708                  */
2709                 if (key.offset != offset)
2710                         continue;
2711
2712                 extent_offset = btrfs_file_extent_offset(leaf, extent);
2713                 num_bytes = btrfs_file_extent_num_bytes(leaf, extent);
2714
2715                 if (extent_offset >= old->extent_offset + old->offset +
2716                     old->len || extent_offset + num_bytes <=
2717                     old->extent_offset + old->offset)
2718                         continue;
2719                 break;
2720         }
2721
2722         backref = kmalloc(sizeof(*backref), GFP_NOFS);
2723         if (!backref) {
2724                 ret = -ENOENT;
2725                 goto out;
2726         }
2727
2728         backref->root_id = root_id;
2729         backref->inum = inum;
2730         backref->file_pos = offset;
2731         backref->num_bytes = num_bytes;
2732         backref->extent_offset = extent_offset;
2733         backref->generation = btrfs_file_extent_generation(leaf, extent);
2734         backref->old = old;
2735         backref_insert(&new->root, backref);
2736         old->count++;
2737 out:
2738         btrfs_release_path(path);
2739         WARN_ON(ret);
2740         return ret;
2741 }
2742
2743 static noinline bool record_extent_backrefs(struct btrfs_path *path,
2744                                    struct new_sa_defrag_extent *new)
2745 {
2746         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2747         struct old_sa_defrag_extent *old, *tmp;
2748         int ret;
2749
2750         new->path = path;
2751
2752         list_for_each_entry_safe(old, tmp, &new->head, list) {
2753                 ret = iterate_inodes_from_logical(old->bytenr +
2754                                                   old->extent_offset, fs_info,
2755                                                   path, record_one_backref,
2756                                                   old, false);
2757                 if (ret < 0 && ret != -ENOENT)
2758                         return false;
2759
2760                 /* no backref to be processed for this extent */
2761                 if (!old->count) {
2762                         list_del(&old->list);
2763                         kfree(old);
2764                 }
2765         }
2766
2767         if (list_empty(&new->head))
2768                 return false;
2769
2770         return true;
2771 }
2772
2773 static int relink_is_mergable(struct extent_buffer *leaf,
2774                               struct btrfs_file_extent_item *fi,
2775                               struct new_sa_defrag_extent *new)
2776 {
2777         if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr)
2778                 return 0;
2779
2780         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2781                 return 0;
2782
2783         if (btrfs_file_extent_compression(leaf, fi) != new->compress_type)
2784                 return 0;
2785
2786         if (btrfs_file_extent_encryption(leaf, fi) ||
2787             btrfs_file_extent_other_encoding(leaf, fi))
2788                 return 0;
2789
2790         return 1;
2791 }
2792
2793 /*
2794  * Note the backref might has changed, and in this case we just return 0.
2795  */
2796 static noinline int relink_extent_backref(struct btrfs_path *path,
2797                                  struct sa_defrag_extent_backref *prev,
2798                                  struct sa_defrag_extent_backref *backref)
2799 {
2800         struct btrfs_file_extent_item *extent;
2801         struct btrfs_file_extent_item *item;
2802         struct btrfs_ordered_extent *ordered;
2803         struct btrfs_trans_handle *trans;
2804         struct btrfs_ref ref = { 0 };
2805         struct btrfs_root *root;
2806         struct btrfs_key key;
2807         struct extent_buffer *leaf;
2808         struct old_sa_defrag_extent *old = backref->old;
2809         struct new_sa_defrag_extent *new = old->new;
2810         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2811         struct inode *inode;
2812         struct extent_state *cached = NULL;
2813         int ret = 0;
2814         u64 start;
2815         u64 len;
2816         u64 lock_start;
2817         u64 lock_end;
2818         bool merge = false;
2819         int index;
2820
2821         if (prev && prev->root_id == backref->root_id &&
2822             prev->inum == backref->inum &&
2823             prev->file_pos + prev->num_bytes == backref->file_pos)
2824                 merge = true;
2825
2826         /* step 1: get root */
2827         key.objectid = backref->root_id;
2828         key.type = BTRFS_ROOT_ITEM_KEY;
2829         key.offset = (u64)-1;
2830
2831         index = srcu_read_lock(&fs_info->subvol_srcu);
2832
2833         root = btrfs_read_fs_root_no_name(fs_info, &key);
2834         if (IS_ERR(root)) {
2835                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2836                 if (PTR_ERR(root) == -ENOENT)
2837                         return 0;
2838                 return PTR_ERR(root);
2839         }
2840
2841         if (btrfs_root_readonly(root)) {
2842                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2843                 return 0;
2844         }
2845
2846         /* step 2: get inode */
2847         key.objectid = backref->inum;
2848         key.type = BTRFS_INODE_ITEM_KEY;
2849         key.offset = 0;
2850
2851         inode = btrfs_iget(fs_info->sb, &key, root, NULL);
2852         if (IS_ERR(inode)) {
2853                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2854                 return 0;
2855         }
2856
2857         srcu_read_unlock(&fs_info->subvol_srcu, index);
2858
2859         /* step 3: relink backref */
2860         lock_start = backref->file_pos;
2861         lock_end = backref->file_pos + backref->num_bytes - 1;
2862         lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2863                          &cached);
2864
2865         ordered = btrfs_lookup_first_ordered_extent(inode, lock_end);
2866         if (ordered) {
2867                 btrfs_put_ordered_extent(ordered);
2868                 goto out_unlock;
2869         }
2870
2871         trans = btrfs_join_transaction(root);
2872         if (IS_ERR(trans)) {
2873                 ret = PTR_ERR(trans);
2874                 goto out_unlock;
2875         }
2876
2877         key.objectid = backref->inum;
2878         key.type = BTRFS_EXTENT_DATA_KEY;
2879         key.offset = backref->file_pos;
2880
2881         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2882         if (ret < 0) {
2883                 goto out_free_path;
2884         } else if (ret > 0) {
2885                 ret = 0;
2886                 goto out_free_path;
2887         }
2888
2889         extent = btrfs_item_ptr(path->nodes[0], path->slots[0],
2890                                 struct btrfs_file_extent_item);
2891
2892         if (btrfs_file_extent_generation(path->nodes[0], extent) !=
2893             backref->generation)
2894                 goto out_free_path;
2895
2896         btrfs_release_path(path);
2897
2898         start = backref->file_pos;
2899         if (backref->extent_offset < old->extent_offset + old->offset)
2900                 start += old->extent_offset + old->offset -
2901                          backref->extent_offset;
2902
2903         len = min(backref->extent_offset + backref->num_bytes,
2904                   old->extent_offset + old->offset + old->len);
2905         len -= max(backref->extent_offset, old->extent_offset + old->offset);
2906
2907         ret = btrfs_drop_extents(trans, root, inode, start,
2908                                  start + len, 1);
2909         if (ret)
2910                 goto out_free_path;
2911 again:
2912         key.objectid = btrfs_ino(BTRFS_I(inode));
2913         key.type = BTRFS_EXTENT_DATA_KEY;
2914         key.offset = start;
2915
2916         path->leave_spinning = 1;
2917         if (merge) {
2918                 struct btrfs_file_extent_item *fi;
2919                 u64 extent_len;
2920                 struct btrfs_key found_key;
2921
2922                 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2923                 if (ret < 0)
2924                         goto out_free_path;
2925
2926                 path->slots[0]--;
2927                 leaf = path->nodes[0];
2928                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2929
2930                 fi = btrfs_item_ptr(leaf, path->slots[0],
2931                                     struct btrfs_file_extent_item);
2932                 extent_len = btrfs_file_extent_num_bytes(leaf, fi);
2933
2934                 if (extent_len + found_key.offset == start &&
2935                     relink_is_mergable(leaf, fi, new)) {
2936                         btrfs_set_file_extent_num_bytes(leaf, fi,
2937                                                         extent_len + len);
2938                         btrfs_mark_buffer_dirty(leaf);
2939                         inode_add_bytes(inode, len);
2940
2941                         ret = 1;
2942                         goto out_free_path;
2943                 } else {
2944                         merge = false;
2945                         btrfs_release_path(path);
2946                         goto again;
2947                 }
2948         }
2949
2950         ret = btrfs_insert_empty_item(trans, root, path, &key,
2951                                         sizeof(*extent));
2952         if (ret) {
2953                 btrfs_abort_transaction(trans, ret);
2954                 goto out_free_path;
2955         }
2956
2957         leaf = path->nodes[0];
2958         item = btrfs_item_ptr(leaf, path->slots[0],
2959                                 struct btrfs_file_extent_item);
2960         btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr);
2961         btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len);
2962         btrfs_set_file_extent_offset(leaf, item, start - new->file_pos);
2963         btrfs_set_file_extent_num_bytes(leaf, item, len);
2964         btrfs_set_file_extent_ram_bytes(leaf, item, new->len);
2965         btrfs_set_file_extent_generation(leaf, item, trans->transid);
2966         btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
2967         btrfs_set_file_extent_compression(leaf, item, new->compress_type);
2968         btrfs_set_file_extent_encryption(leaf, item, 0);
2969         btrfs_set_file_extent_other_encoding(leaf, item, 0);
2970
2971         btrfs_mark_buffer_dirty(leaf);
2972         inode_add_bytes(inode, len);
2973         btrfs_release_path(path);
2974
2975         btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF, new->bytenr,
2976                                new->disk_len, 0);
2977         btrfs_init_data_ref(&ref, backref->root_id, backref->inum,
2978                             new->file_pos);  /* start - extent_offset */
2979         ret = btrfs_inc_extent_ref(trans, &ref);
2980         if (ret) {
2981                 btrfs_abort_transaction(trans, ret);
2982                 goto out_free_path;
2983         }
2984
2985         ret = 1;
2986 out_free_path:
2987         btrfs_release_path(path);
2988         path->leave_spinning = 0;
2989         btrfs_end_transaction(trans);
2990 out_unlock:
2991         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2992                              &cached);
2993         iput(inode);
2994         return ret;
2995 }
2996
2997 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new)
2998 {
2999         struct old_sa_defrag_extent *old, *tmp;
3000
3001         if (!new)
3002                 return;
3003
3004         list_for_each_entry_safe(old, tmp, &new->head, list) {
3005                 kfree(old);
3006         }
3007         kfree(new);
3008 }
3009
3010 static void relink_file_extents(struct new_sa_defrag_extent *new)
3011 {
3012         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
3013         struct btrfs_path *path;
3014         struct sa_defrag_extent_backref *backref;
3015         struct sa_defrag_extent_backref *prev = NULL;
3016         struct rb_node *node;
3017         int ret;
3018
3019         path = btrfs_alloc_path();
3020         if (!path)
3021                 return;
3022
3023         if (!record_extent_backrefs(path, new)) {
3024                 btrfs_free_path(path);
3025                 goto out;
3026         }
3027         btrfs_release_path(path);
3028
3029         while (1) {
3030                 node = rb_first(&new->root);
3031                 if (!node)
3032                         break;
3033                 rb_erase(node, &new->root);
3034
3035                 backref = rb_entry(node, struct sa_defrag_extent_backref, node);
3036
3037                 ret = relink_extent_backref(path, prev, backref);
3038                 WARN_ON(ret < 0);
3039
3040                 kfree(prev);
3041
3042                 if (ret == 1)
3043                         prev = backref;
3044                 else
3045                         prev = NULL;
3046                 cond_resched();
3047         }
3048         kfree(prev);
3049
3050         btrfs_free_path(path);
3051 out:
3052         free_sa_defrag_extent(new);
3053
3054         atomic_dec(&fs_info->defrag_running);
3055         wake_up(&fs_info->transaction_wait);
3056 }
3057
3058 static struct new_sa_defrag_extent *
3059 record_old_file_extents(struct inode *inode,
3060                         struct btrfs_ordered_extent *ordered)
3061 {
3062         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3063         struct btrfs_root *root = BTRFS_I(inode)->root;
3064         struct btrfs_path *path;
3065         struct btrfs_key key;
3066         struct old_sa_defrag_extent *old;
3067         struct new_sa_defrag_extent *new;
3068         int ret;
3069
3070         new = kmalloc(sizeof(*new), GFP_NOFS);
3071         if (!new)
3072                 return NULL;
3073
3074         new->inode = inode;
3075         new->file_pos = ordered->file_offset;
3076         new->len = ordered->len;
3077         new->bytenr = ordered->start;
3078         new->disk_len = ordered->disk_len;
3079         new->compress_type = ordered->compress_type;
3080         new->root = RB_ROOT;
3081         INIT_LIST_HEAD(&new->head);
3082
3083         path = btrfs_alloc_path();
3084         if (!path)
3085                 goto out_kfree;
3086
3087         key.objectid = btrfs_ino(BTRFS_I(inode));
3088         key.type = BTRFS_EXTENT_DATA_KEY;
3089         key.offset = new->file_pos;
3090
3091         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3092         if (ret < 0)
3093                 goto out_free_path;
3094         if (ret > 0 && path->slots[0] > 0)
3095                 path->slots[0]--;
3096
3097         /* find out all the old extents for the file range */
3098         while (1) {
3099                 struct btrfs_file_extent_item *extent;
3100                 struct extent_buffer *l;
3101                 int slot;
3102                 u64 num_bytes;
3103                 u64 offset;
3104                 u64 end;
3105                 u64 disk_bytenr;
3106                 u64 extent_offset;
3107
3108                 l = path->nodes[0];
3109                 slot = path->slots[0];
3110
3111                 if (slot >= btrfs_header_nritems(l)) {
3112                         ret = btrfs_next_leaf(root, path);
3113                         if (ret < 0)
3114                                 goto out_free_path;
3115                         else if (ret > 0)
3116                                 break;
3117                         continue;
3118                 }
3119
3120                 btrfs_item_key_to_cpu(l, &key, slot);
3121
3122                 if (key.objectid != btrfs_ino(BTRFS_I(inode)))
3123                         break;
3124                 if (key.type != BTRFS_EXTENT_DATA_KEY)
3125                         break;
3126                 if (key.offset >= new->file_pos + new->len)
3127                         break;
3128
3129                 extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item);
3130
3131                 num_bytes = btrfs_file_extent_num_bytes(l, extent);
3132                 if (key.offset + num_bytes < new->file_pos)
3133                         goto next;
3134
3135                 disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent);
3136                 if (!disk_bytenr)
3137                         goto next;
3138
3139                 extent_offset = btrfs_file_extent_offset(l, extent);
3140
3141                 old = kmalloc(sizeof(*old), GFP_NOFS);
3142                 if (!old)
3143                         goto out_free_path;
3144
3145                 offset = max(new->file_pos, key.offset);
3146                 end = min(new->file_pos + new->len, key.offset + num_bytes);
3147
3148                 old->bytenr = disk_bytenr;
3149                 old->extent_offset = extent_offset;
3150                 old->offset = offset - key.offset;
3151                 old->len = end - offset;
3152                 old->new = new;
3153                 old->count = 0;
3154                 list_add_tail(&old->list, &new->head);
3155 next:
3156                 path->slots[0]++;
3157                 cond_resched();
3158         }
3159
3160         btrfs_free_path(path);
3161         atomic_inc(&fs_info->defrag_running);
3162
3163         return new;
3164
3165 out_free_path:
3166         btrfs_free_path(path);
3167 out_kfree:
3168         free_sa_defrag_extent(new);
3169         return NULL;
3170 }
3171
3172 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,
3173                                          u64 start, u64 len)
3174 {
3175         struct btrfs_block_group_cache *cache;
3176
3177         cache = btrfs_lookup_block_group(fs_info, start);
3178         ASSERT(cache);
3179
3180         spin_lock(&cache->lock);
3181         cache->delalloc_bytes -= len;
3182         spin_unlock(&cache->lock);
3183
3184         btrfs_put_block_group(cache);
3185 }
3186
3187 /* as ordered data IO finishes, this gets called so we can finish
3188  * an ordered extent if the range of bytes in the file it covers are
3189  * fully written.
3190  */
3191 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
3192 {
3193         struct inode *inode = ordered_extent->inode;
3194         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3195         struct btrfs_root *root = BTRFS_I(inode)->root;
3196         struct btrfs_trans_handle *trans = NULL;
3197         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
3198         struct extent_state *cached_state = NULL;
3199         struct new_sa_defrag_extent *new = NULL;
3200         int compress_type = 0;
3201         int ret = 0;
3202         u64 logical_len = ordered_extent->len;
3203         bool nolock;
3204         bool truncated = false;
3205         bool range_locked = false;
3206         bool clear_new_delalloc_bytes = false;
3207         bool clear_reserved_extent = true;
3208
3209         if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3210             !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) &&
3211             !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags))
3212                 clear_new_delalloc_bytes = true;
3213
3214         nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
3215
3216         if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
3217                 ret = -EIO;
3218                 goto out;
3219         }
3220
3221         btrfs_free_io_failure_record(BTRFS_I(inode),
3222                         ordered_extent->file_offset,
3223                         ordered_extent->file_offset +
3224                         ordered_extent->len - 1);
3225
3226         if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
3227                 truncated = true;
3228                 logical_len = ordered_extent->truncated_len;
3229                 /* Truncated the entire extent, don't bother adding */
3230                 if (!logical_len)
3231                         goto out;
3232         }
3233
3234         if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
3235                 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
3236
3237                 /*
3238                  * For mwrite(mmap + memset to write) case, we still reserve
3239                  * space for NOCOW range.
3240                  * As NOCOW won't cause a new delayed ref, just free the space
3241                  */
3242                 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset,
3243                                        ordered_extent->len);
3244                 btrfs_ordered_update_i_size(inode, 0, ordered_extent);
3245                 if (nolock)
3246                         trans = btrfs_join_transaction_nolock(root);
3247                 else
3248                         trans = btrfs_join_transaction(root);
3249                 if (IS_ERR(trans)) {
3250                         ret = PTR_ERR(trans);
3251                         trans = NULL;
3252                         goto out;
3253                 }
3254                 trans->block_rsv = &BTRFS_I(inode)->block_rsv;
3255                 ret = btrfs_update_inode_fallback(trans, root, inode);
3256                 if (ret) /* -ENOMEM or corruption */
3257                         btrfs_abort_transaction(trans, ret);
3258                 goto out;
3259         }
3260
3261         range_locked = true;
3262         lock_extent_bits(io_tree, ordered_extent->file_offset,
3263                          ordered_extent->file_offset + ordered_extent->len - 1,
3264                          &cached_state);
3265
3266         ret = test_range_bit(io_tree, ordered_extent->file_offset,
3267                         ordered_extent->file_offset + ordered_extent->len - 1,
3268                         EXTENT_DEFRAG, 0, cached_state);
3269         if (ret) {
3270                 u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item);
3271                 if (0 && last_snapshot >= BTRFS_I(inode)->generation)
3272                         /* the inode is shared */
3273                         new = record_old_file_extents(inode, ordered_extent);
3274
3275                 clear_extent_bit(io_tree, ordered_extent->file_offset,
3276                         ordered_extent->file_offset + ordered_extent->len - 1,
3277                         EXTENT_DEFRAG, 0, 0, &cached_state);
3278         }
3279
3280         if (nolock)
3281                 trans = btrfs_join_transaction_nolock(root);
3282         else
3283                 trans = btrfs_join_transaction(root);
3284         if (IS_ERR(trans)) {
3285                 ret = PTR_ERR(trans);
3286                 trans = NULL;
3287                 goto out;
3288         }
3289
3290         trans->block_rsv = &BTRFS_I(inode)->block_rsv;
3291
3292         if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
3293                 compress_type = ordered_extent->compress_type;
3294         if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
3295                 BUG_ON(compress_type);
3296                 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset,
3297                                        ordered_extent->len);
3298                 ret = btrfs_mark_extent_written(trans, BTRFS_I(inode),
3299                                                 ordered_extent->file_offset,
3300                                                 ordered_extent->file_offset +
3301                                                 logical_len);
3302         } else {
3303                 BUG_ON(root == fs_info->tree_root);
3304                 ret = insert_reserved_file_extent(trans, inode,
3305                                                 ordered_extent->file_offset,
3306                                                 ordered_extent->start,
3307                                                 ordered_extent->disk_len,
3308                                                 logical_len, logical_len,
3309                                                 compress_type, 0, 0,
3310                                                 BTRFS_FILE_EXTENT_REG);
3311                 if (!ret) {
3312                         clear_reserved_extent = false;
3313                         btrfs_release_delalloc_bytes(fs_info,
3314                                                      ordered_extent->start,
3315                                                      ordered_extent->disk_len);
3316                 }
3317         }
3318         unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
3319                            ordered_extent->file_offset, ordered_extent->len,
3320                            trans->transid);
3321         if (ret < 0) {
3322                 btrfs_abort_transaction(trans, ret);
3323                 goto out;
3324         }
3325
3326         ret = add_pending_csums(trans, inode, &ordered_extent->list);
3327         if (ret) {
3328                 btrfs_abort_transaction(trans, ret);
3329                 goto out;
3330         }
3331
3332         btrfs_ordered_update_i_size(inode, 0, ordered_extent);
3333         ret = btrfs_update_inode_fallback(trans, root, inode);
3334         if (ret) { /* -ENOMEM or corruption */
3335                 btrfs_abort_transaction(trans, ret);
3336                 goto out;
3337         }
3338         ret = 0;
3339 out:
3340         if (range_locked || clear_new_delalloc_bytes) {
3341                 unsigned int clear_bits = 0;
3342
3343                 if (range_locked)
3344                         clear_bits |= EXTENT_LOCKED;
3345                 if (clear_new_delalloc_bytes)
3346                         clear_bits |= EXTENT_DELALLOC_NEW;
3347                 clear_extent_bit(&BTRFS_I(inode)->io_tree,
3348                                  ordered_extent->file_offset,
3349                                  ordered_extent->file_offset +
3350                                  ordered_extent->len - 1,
3351                                  clear_bits,
3352                                  (clear_bits & EXTENT_LOCKED) ? 1 : 0,
3353                                  0, &cached_state);
3354         }
3355
3356         if (trans)
3357                 btrfs_end_transaction(trans);
3358
3359         if (ret || truncated) {
3360                 u64 start, end;
3361
3362                 /*
3363                  * If we failed to finish this ordered extent for any reason we
3364                  * need to make sure BTRFS_ORDERED_IOERR is set on the ordered
3365                  * extent, and mark the inode with the error if it wasn't
3366                  * already set.  Any error during writeback would have already
3367                  * set the mapping error, so we need to set it if we're the ones
3368                  * marking this ordered extent as failed.
3369                  */
3370                 if (ret && !test_and_set_bit(BTRFS_ORDERED_IOERR,
3371                                              &ordered_extent->flags))
3372                         mapping_set_error(ordered_extent->inode->i_mapping, -EIO);
3373
3374                 if (truncated)
3375                         start = ordered_extent->file_offset + logical_len;
3376                 else
3377                         start = ordered_extent->file_offset;
3378                 end = ordered_extent->file_offset + ordered_extent->len - 1;
3379                 clear_extent_uptodate(io_tree, start, end, NULL);
3380
3381                 /* Drop the cache for the part of the extent we didn't write. */
3382                 btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0);
3383
3384                 /*
3385                  * If the ordered extent had an IOERR or something else went
3386                  * wrong we need to return the space for this ordered extent
3387                  * back to the allocator.  We only free the extent in the
3388                  * truncated case if we didn't write out the extent at all.
3389                  *
3390                  * If we made it past insert_reserved_file_extent before we
3391                  * errored out then we don't need to do this as the accounting
3392                  * has already been done.
3393                  */
3394                 if ((ret || !logical_len) &&
3395                     clear_reserved_extent &&
3396                     !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3397                     !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags))
3398                         btrfs_free_reserved_extent(fs_info,
3399                                                    ordered_extent->start,
3400                                                    ordered_extent->disk_len, 1);
3401         }
3402
3403
3404         /*
3405          * This needs to be done to make sure anybody waiting knows we are done
3406          * updating everything for this ordered extent.
3407          */
3408         btrfs_remove_ordered_extent(inode, ordered_extent);
3409
3410         /* for snapshot-aware defrag */
3411         if (new) {
3412                 if (ret) {
3413                         free_sa_defrag_extent(new);
3414                         atomic_dec(&fs_info->defrag_running);
3415                 } else {
3416                         relink_file_extents(new);
3417                 }
3418         }
3419
3420         /* once for us */
3421         btrfs_put_ordered_extent(ordered_extent);
3422         /* once for the tree */
3423         btrfs_put_ordered_extent(ordered_extent);
3424
3425         return ret;
3426 }
3427
3428 static void finish_ordered_fn(struct btrfs_work *work)
3429 {
3430         struct btrfs_ordered_extent *ordered_extent;
3431         ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
3432         btrfs_finish_ordered_io(ordered_extent);
3433 }
3434
3435 void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start,
3436                                           u64 end, int uptodate)
3437 {
3438         struct inode *inode = page->mapping->host;
3439         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3440         struct btrfs_ordered_extent *ordered_extent = NULL;
3441         struct btrfs_workqueue *wq;
3442
3443         trace_btrfs_writepage_end_io_hook(page, start, end, uptodate);
3444
3445         ClearPagePrivate2(page);
3446         if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
3447                                             end - start + 1, uptodate))
3448                 return;
3449
3450         if (btrfs_is_free_space_inode(BTRFS_I(inode)))
3451                 wq = fs_info->endio_freespace_worker;
3452         else
3453                 wq = fs_info->endio_write_workers;
3454
3455         btrfs_init_work(&ordered_extent->work, finish_ordered_fn, NULL, NULL);
3456         btrfs_queue_work(wq, &ordered_extent->work);
3457 }
3458
3459 static int __readpage_endio_check(struct inode *inode,
3460                                   struct btrfs_io_bio *io_bio,
3461                                   int icsum, struct page *page,
3462                                   int pgoff, u64 start, size_t len)
3463 {
3464         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3465         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
3466         char *kaddr;
3467         u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
3468         u8 *csum_expected;
3469         u8 csum[BTRFS_CSUM_SIZE];
3470
3471         csum_expected = ((u8 *)io_bio->csum) + icsum * csum_size;
3472
3473         kaddr = kmap_atomic(page);
3474         shash->tfm = fs_info->csum_shash;
3475
3476         crypto_shash_init(shash);
3477         crypto_shash_update(shash, kaddr + pgoff, len);
3478         crypto_shash_final(shash, csum);
3479
3480         if (memcmp(csum, csum_expected, csum_size))
3481                 goto zeroit;
3482
3483         kunmap_atomic(kaddr);
3484         return 0;
3485 zeroit:
3486         btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected,
3487                                     io_bio->mirror_num);
3488         memset(kaddr + pgoff, 1, len);
3489         flush_dcache_page(page);
3490         kunmap_atomic(kaddr);
3491         return -EIO;
3492 }
3493
3494 /*
3495  * when reads are done, we need to check csums to verify the data is correct
3496  * if there's a match, we allow the bio to finish.  If not, the code in
3497  * extent_io.c will try to find good copies for us.
3498  */
3499 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
3500                                       u64 phy_offset, struct page *page,
3501                                       u64 start, u64 end, int mirror)
3502 {
3503         size_t offset = start - page_offset(page);
3504         struct inode *inode = page->mapping->host;
3505         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
3506         struct btrfs_root *root = BTRFS_I(inode)->root;
3507
3508         if (PageChecked(page)) {
3509                 ClearPageChecked(page);
3510                 return 0;
3511         }
3512
3513         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
3514                 return 0;
3515
3516         if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
3517             test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) {
3518                 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM);
3519                 return 0;
3520         }
3521
3522         phy_offset >>= inode->i_sb->s_blocksize_bits;
3523         return __readpage_endio_check(inode, io_bio, phy_offset, page, offset,
3524                                       start, (size_t)(end - start + 1));
3525 }
3526
3527 /*
3528  * btrfs_add_delayed_iput - perform a delayed iput on @inode
3529  *
3530  * @inode: The inode we want to perform iput on
3531  *
3532  * This function uses the generic vfs_inode::i_count to track whether we should
3533  * just decrement it (in case it's > 1) or if this is the last iput then link
3534  * the inode to the delayed iput machinery. Delayed iputs are processed at
3535  * transaction commit time/superblock commit/cleaner kthread.
3536  */
3537 void btrfs_add_delayed_iput(struct inode *inode)
3538 {
3539         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3540         struct btrfs_inode *binode = BTRFS_I(inode);
3541
3542         if (atomic_add_unless(&inode->i_count, -1, 1))
3543                 return;
3544
3545         atomic_inc(&fs_info->nr_delayed_iputs);
3546         spin_lock(&fs_info->delayed_iput_lock);
3547         ASSERT(list_empty(&binode->delayed_iput));
3548         list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs);
3549         spin_unlock(&fs_info->delayed_iput_lock);
3550         if (!test_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags))
3551                 wake_up_process(fs_info->cleaner_kthread);
3552 }
3553
3554 static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info,
3555                                     struct btrfs_inode *inode)
3556 {
3557         list_del_init(&inode->delayed_iput);
3558         spin_unlock(&fs_info->delayed_iput_lock);
3559         iput(&inode->vfs_inode);
3560         if (atomic_dec_and_test(&fs_info->nr_delayed_iputs))
3561                 wake_up(&fs_info->delayed_iputs_wait);
3562         spin_lock(&fs_info->delayed_iput_lock);
3563 }
3564
3565 static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info,
3566                                    struct btrfs_inode *inode)
3567 {
3568         if (!list_empty(&inode->delayed_iput)) {
3569                 spin_lock(&fs_info->delayed_iput_lock);
3570                 if (!list_empty(&inode->delayed_iput))
3571                         run_delayed_iput_locked(fs_info, inode);
3572                 spin_unlock(&fs_info->delayed_iput_lock);
3573         }
3574 }
3575
3576 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)
3577 {
3578
3579         spin_lock(&fs_info->delayed_iput_lock);
3580         while (!list_empty(&fs_info->delayed_iputs)) {
3581                 struct btrfs_inode *inode;
3582
3583                 inode = list_first_entry(&fs_info->delayed_iputs,
3584                                 struct btrfs_inode, delayed_iput);
3585                 run_delayed_iput_locked(fs_info, inode);
3586                 cond_resched_lock(&fs_info->delayed_iput_lock);
3587         }
3588         spin_unlock(&fs_info->delayed_iput_lock);
3589 }
3590
3591 /**
3592  * btrfs_wait_on_delayed_iputs - wait on the delayed iputs to be done running
3593  * @fs_info - the fs_info for this fs
3594  * @return - EINTR if we were killed, 0 if nothing's pending
3595  *
3596  * This will wait on any delayed iputs that are currently running with KILLABLE
3597  * set.  Once they are all done running we will return, unless we are killed in
3598  * which case we return EINTR. This helps in user operations like fallocate etc
3599  * that might get blocked on the iputs.
3600  */
3601 int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info)
3602 {
3603         int ret = wait_event_killable(fs_info->delayed_iputs_wait,
3604                         atomic_read(&fs_info->nr_delayed_iputs) == 0);
3605         if (ret)
3606                 return -EINTR;
3607         return 0;
3608 }
3609
3610 /*
3611  * This creates an orphan entry for the given inode in case something goes wrong
3612  * in the middle of an unlink.
3613  */
3614 int btrfs_orphan_add(struct btrfs_trans_handle *trans,
3615                      struct btrfs_inode *inode)
3616 {
3617         int ret;
3618
3619         ret = btrfs_insert_orphan_item(trans, inode->root, btrfs_ino(inode));
3620         if (ret && ret != -EEXIST) {
3621                 btrfs_abort_transaction(trans, ret);
3622                 return ret;
3623         }
3624
3625         return 0;
3626 }
3627
3628 /*
3629  * We have done the delete so we can go ahead and remove the orphan item for
3630  * this particular inode.
3631  */
3632 static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
3633                             struct btrfs_inode *inode)
3634 {
3635         return btrfs_del_orphan_item(trans, inode->root, btrfs_ino(inode));
3636 }
3637
3638 /*
3639  * this cleans up any orphans that may be left on the list from the last use
3640  * of this root.
3641  */
3642 int btrfs_orphan_cleanup(struct btrfs_root *root)
3643 {
3644         struct btrfs_fs_info *fs_info = root->fs_info;
3645         struct btrfs_path *path;
3646         struct extent_buffer *leaf;
3647         struct btrfs_key key, found_key;
3648         struct btrfs_trans_handle *trans;
3649         struct inode *inode;
3650         u64 last_objectid = 0;
3651         int ret = 0, nr_unlink = 0;
3652
3653         if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED))
3654                 return 0;
3655
3656         path = btrfs_alloc_path();
3657         if (!path) {
3658                 ret = -ENOMEM;
3659                 goto out;
3660         }
3661         path->reada = READA_BACK;
3662
3663         key.objectid = BTRFS_ORPHAN_OBJECTID;
3664         key.type = BTRFS_ORPHAN_ITEM_KEY;
3665         key.offset = (u64)-1;
3666
3667         while (1) {
3668                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3669                 if (ret < 0)
3670                         goto out;
3671
3672                 /*
3673                  * if ret == 0 means we found what we were searching for, which
3674                  * is weird, but possible, so only screw with path if we didn't
3675                  * find the key and see if we have stuff that matches
3676                  */
3677                 if (ret > 0) {
3678                         ret = 0;
3679                         if (path->slots[0] == 0)
3680                                 break;
3681                         path->slots[0]--;
3682                 }
3683
3684                 /* pull out the item */
3685                 leaf = path->nodes[0];
3686                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3687
3688                 /* make sure the item matches what we want */
3689                 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
3690                         break;
3691                 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
3692                         break;
3693
3694                 /* release the path since we're done with it */
3695                 btrfs_release_path(path);
3696
3697                 /*
3698                  * this is where we are basically btrfs_lookup, without the
3699                  * crossing root thing.  we store the inode number in the
3700                  * offset of the orphan item.
3701                  */
3702
3703                 if (found_key.offset == last_objectid) {
3704                         btrfs_err(fs_info,
3705                                   "Error removing orphan entry, stopping orphan cleanup");
3706                         ret = -EINVAL;
3707                         goto out;
3708                 }
3709
3710                 last_objectid = found_key.offset;
3711
3712                 found_key.objectid = found_key.offset;
3713                 found_key.type = BTRFS_INODE_ITEM_KEY;
3714                 found_key.offset = 0;
3715                 inode = btrfs_iget(fs_info->sb, &found_key, root, NULL);
3716                 ret = PTR_ERR_OR_ZERO(inode);
3717                 if (ret && ret != -ENOENT)
3718                         goto out;
3719
3720                 if (ret == -ENOENT && root == fs_info->tree_root) {
3721                         struct btrfs_root *dead_root;
3722                         struct btrfs_fs_info *fs_info = root->fs_info;
3723                         int is_dead_root = 0;
3724
3725                         /*
3726                          * this is an orphan in the tree root. Currently these
3727                          * could come from 2 sources:
3728                          *  a) a snapshot deletion in progress
3729                          *  b) a free space cache inode
3730                          * We need to distinguish those two, as the snapshot
3731                          * orphan must not get deleted.
3732                          * find_dead_roots already ran before us, so if this
3733                          * is a snapshot deletion, we should find the root
3734                          * in the dead_roots list
3735                          */
3736                         spin_lock(&fs_info->trans_lock);
3737                         list_for_each_entry(dead_root, &fs_info->dead_roots,
3738                                             root_list) {
3739                                 if (dead_root->root_key.objectid ==
3740                                     found_key.objectid) {
3741                                         is_dead_root = 1;
3742                                         break;
3743                                 }
3744                         }
3745                         spin_unlock(&fs_info->trans_lock);
3746                         if (is_dead_root) {
3747                                 /* prevent this orphan from being found again */
3748                                 key.offset = found_key.objectid - 1;
3749                                 continue;
3750                         }
3751
3752                 }
3753
3754                 /*
3755                  * If we have an inode with links, there are a couple of
3756                  * possibilities. Old kernels (before v3.12) used to create an
3757                  * orphan item for truncate indicating that there were possibly
3758                  * extent items past i_size that needed to be deleted. In v3.12,
3759                  * truncate was changed to update i_size in sync with the extent
3760                  * items, but the (useless) orphan item was still created. Since
3761                  * v4.18, we don't create the orphan item for truncate at all.
3762                  *
3763                  * So, this item could mean that we need to do a truncate, but
3764                  * only if this filesystem was last used on a pre-v3.12 kernel
3765                  * and was not cleanly unmounted. The odds of that are quite
3766                  * slim, and it's a pain to do the truncate now, so just delete
3767                  * the orphan item.
3768                  *
3769                  * It's also possible that this orphan item was supposed to be
3770                  * deleted but wasn't. The inode number may have been reused,
3771                  * but either way, we can delete the orphan item.
3772                  */
3773                 if (ret == -ENOENT || inode->i_nlink) {
3774                         if (!ret)
3775                                 iput(inode);
3776                         trans = btrfs_start_transaction(root, 1);
3777                         if (IS_ERR(trans)) {
3778                                 ret = PTR_ERR(trans);
3779                                 goto out;
3780                         }
3781                         btrfs_debug(fs_info, "auto deleting %Lu",
3782                                     found_key.objectid);
3783                         ret = btrfs_del_orphan_item(trans, root,
3784                                                     found_key.objectid);
3785                         btrfs_end_transaction(trans);
3786                         if (ret)
3787                                 goto out;
3788                         continue;
3789                 }
3790
3791                 nr_unlink++;
3792
3793                 /* this will do delete_inode and everything for us */
3794                 iput(inode);
3795         }
3796         /* release the path since we're done with it */
3797         btrfs_release_path(path);
3798
3799         root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE;
3800
3801         if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
3802                 trans = btrfs_join_transaction(root);
3803                 if (!IS_ERR(trans))
3804                         btrfs_end_transaction(trans);
3805         }
3806
3807         if (nr_unlink)
3808                 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);
3809
3810 out:
3811         if (ret)
3812                 btrfs_err(fs_info, "could not do orphan cleanup %d", ret);
3813         btrfs_free_path(path);
3814         return ret;
3815 }
3816
3817 /*
3818  * very simple check to peek ahead in the leaf looking for xattrs.  If we
3819  * don't find any xattrs, we know there can't be any acls.
3820  *
3821  * slot is the slot the inode is in, objectid is the objectid of the inode
3822  */
3823 static noinline int acls_after_inode_item(struct extent_buffer *leaf,
3824                                           int slot, u64 objectid,
3825                                           int *first_xattr_slot)
3826 {
3827         u32 nritems = btrfs_header_nritems(leaf);
3828         struct btrfs_key found_key;
3829         static u64 xattr_access = 0;
3830         static u64 xattr_default = 0;
3831         int scanned = 0;
3832
3833         if (!xattr_access) {
3834                 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
3835                                         strlen(XATTR_NAME_POSIX_ACL_ACCESS));
3836                 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
3837                                         strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
3838         }
3839
3840         slot++;
3841         *first_xattr_slot = -1;
3842         while (slot < nritems) {
3843                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
3844
3845                 /* we found a different objectid, there must not be acls */
3846                 if (found_key.objectid != objectid)
3847                         return 0;
3848
3849                 /* we found an xattr, assume we've got an acl */
3850                 if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
3851                         if (*first_xattr_slot == -1)
3852                                 *first_xattr_slot = slot;
3853                         if (found_key.offset == xattr_access ||
3854                             found_key.offset == xattr_default)
3855                                 return 1;
3856                 }
3857
3858                 /*
3859                  * we found a key greater than an xattr key, there can't
3860                  * be any acls later on
3861                  */
3862                 if (found_key.type > BTRFS_XATTR_ITEM_KEY)
3863                         return 0;
3864
3865                 slot++;
3866                 scanned++;
3867
3868                 /*
3869                  * it goes inode, inode backrefs, xattrs, extents,
3870                  * so if there are a ton of hard links to an inode there can
3871                  * be a lot of backrefs.  Don't waste time searching too hard,
3872                  * this is just an optimization
3873                  */
3874                 if (scanned >= 8)
3875                         break;
3876         }
3877         /* we hit the end of the leaf before we found an xattr or
3878          * something larger than an xattr.  We have to assume the inode
3879          * has acls
3880          */
3881         if (*first_xattr_slot == -1)
3882                 *first_xattr_slot = slot;
3883         return 1;
3884 }
3885
3886 /*
3887  * read an inode from the btree into the in-memory inode
3888  */
3889 static int btrfs_read_locked_inode(struct inode *inode,
3890                                    struct btrfs_path *in_path)
3891 {
3892         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3893         struct btrfs_path *path = in_path;
3894         struct extent_buffer *leaf;
3895         struct btrfs_inode_item *inode_item;
3896         struct btrfs_root *root = BTRFS_I(inode)->root;
3897         struct btrfs_key location;
3898         unsigned long ptr;
3899         int maybe_acls;
3900         u32 rdev;
3901         int ret;
3902         bool filled = false;
3903         int first_xattr_slot;
3904
3905         ret = btrfs_fill_inode(inode, &rdev);
3906         if (!ret)
3907                 filled = true;
3908
3909         if (!path) {
3910                 path = btrfs_alloc_path();
3911                 if (!path)
3912                         return -ENOMEM;
3913         }
3914
3915         memcpy(&location, &BTRFS_I(inode)->location, sizeof(location));
3916
3917         ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
3918         if (ret) {
3919                 if (path != in_path)
3920                         btrfs_free_path(path);
3921                 return ret;
3922         }
3923
3924         leaf = path->nodes[0];
3925
3926         if (filled)
3927                 goto cache_index;
3928
3929         inode_item = btrfs_item_ptr(leaf, path->slots[0],
3930                                     struct btrfs_inode_item);
3931         inode->i_mode = btrfs_inode_mode(leaf, inode_item);
3932         set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
3933         i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
3934         i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
3935         btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item));
3936
3937         inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime);
3938         inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime);
3939
3940         inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime);
3941         inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime);
3942
3943         inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime);
3944         inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime);
3945
3946         BTRFS_I(inode)->i_otime.tv_sec =
3947                 btrfs_timespec_sec(leaf, &inode_item->otime);
3948         BTRFS_I(inode)->i_otime.tv_nsec =
3949                 btrfs_timespec_nsec(leaf, &inode_item->otime);
3950
3951         inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));
3952         BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);
3953         BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);
3954
3955         inode_set_iversion_queried(inode,
3956                                    btrfs_inode_sequence(leaf, inode_item));
3957         inode->i_generation = BTRFS_I(inode)->generation;
3958         inode->i_rdev = 0;
3959         rdev = btrfs_inode_rdev(leaf, inode_item);
3960
3961         BTRFS_I(inode)->index_cnt = (u64)-1;
3962         BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item);
3963
3964 cache_index:
3965         /*
3966          * If we were modified in the current generation and evicted from memory
3967          * and then re-read we need to do a full sync since we don't have any
3968          * idea about which extents were modified before we were evicted from
3969          * cache.
3970          *
3971          * This is required for both inode re-read from disk and delayed inode
3972          * in delayed_nodes_tree.
3973          */
3974         if (BTRFS_I(inode)->last_trans == fs_info->generation)
3975                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
3976                         &BTRFS_I(inode)->runtime_flags);
3977
3978         /*
3979          * We don't persist the id of the transaction where an unlink operation
3980          * against the inode was last made. So here we assume the inode might
3981          * have been evicted, and therefore the exact value of last_unlink_trans
3982          * lost, and set it to last_trans to avoid metadata inconsistencies
3983          * between the inode and its parent if the inode is fsync'ed and the log
3984          * replayed. For example, in the scenario:
3985          *
3986          * touch mydir/foo
3987          * ln mydir/foo mydir/bar
3988          * sync
3989          * unlink mydir/bar
3990          * echo 2 > /proc/sys/vm/drop_caches   # evicts inode
3991          * xfs_io -c fsync mydir/foo
3992          * <power failure>
3993          * mount fs, triggers fsync log replay
3994          *
3995          * We must make sure that when we fsync our inode foo we also log its
3996          * parent inode, otherwise after log replay the parent still has the
3997          * dentry with the "bar" name but our inode foo has a link count of 1
3998          * and doesn't have an inode ref with the name "bar" anymore.
3999          *
4000          * Setting last_unlink_trans to last_trans is a pessimistic approach,
4001          * but it guarantees correctness at the expense of occasional full
4002          * transaction commits on fsync if our inode is a directory, or if our
4003          * inode is not a directory, logging its parent unnecessarily.
4004          */
4005         BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans;
4006
4007         path->slots[0]++;
4008         if (inode->i_nlink != 1 ||
4009             path->slots[0] >= btrfs_header_nritems(leaf))
4010                 goto cache_acl;
4011
4012         btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
4013         if (location.objectid != btrfs_ino(BTRFS_I(inode)))
4014                 goto cache_acl;
4015
4016         ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
4017         if (location.type == BTRFS_INODE_REF_KEY) {
4018                 struct btrfs_inode_ref *ref;
4019
4020                 ref = (struct btrfs_inode_ref *)ptr;
4021                 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);
4022         } else if (location.type == BTRFS_INODE_EXTREF_KEY) {
4023                 struct btrfs_inode_extref *extref;
4024
4025                 extref = (struct btrfs_inode_extref *)ptr;
4026                 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,
4027                                                                      extref);
4028         }
4029 cache_acl:
4030         /*
4031          * try to precache a NULL acl entry for files that don't have
4032          * any xattrs or acls
4033          */
4034         maybe_acls = acls_after_inode_item(leaf, path->slots[0],
4035                         btrfs_ino(BTRFS_I(inode)), &first_xattr_slot);
4036         if (first_xattr_slot != -1) {
4037                 path->slots[0] = first_xattr_slot;
4038                 ret = btrfs_load_inode_props(inode, path);
4039                 if (ret)
4040                         btrfs_err(fs_info,
4041                                   "error loading props for ino %llu (root %llu): %d",
4042                                   btrfs_ino(BTRFS_I(inode)),
4043                                   root->root_key.objectid, ret);
4044         }
4045         if (path != in_path)
4046                 btrfs_free_path(path);
4047
4048         if (!maybe_acls)
4049                 cache_no_acl(inode);
4050
4051         switch (inode->i_mode & S_IFMT) {
4052         case S_IFREG:
4053                 inode->i_mapping->a_ops = &btrfs_aops;
4054                 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
4055                 inode->i_fop = &btrfs_file_operations;
4056                 inode->i_op = &btrfs_file_inode_operations;
4057                 break;
4058         case S_IFDIR:
4059                 inode->i_fop = &btrfs_dir_file_operations;
4060                 inode->i_op = &btrfs_dir_inode_operations;
4061                 break;
4062         case S_IFLNK:
4063                 inode->i_op = &btrfs_symlink_inode_operations;
4064                 inode_nohighmem(inode);
4065                 inode->i_mapping->a_ops = &btrfs_aops;
4066                 break;
4067         default:
4068                 inode->i_op = &btrfs_special_inode_operations;
4069                 init_special_inode(inode, inode->i_mode, rdev);
4070                 break;
4071         }
4072
4073         btrfs_sync_inode_flags_to_i_flags(inode);
4074         return 0;
4075 }
4076
4077 /*
4078  * given a leaf and an inode, copy the inode fields into the leaf
4079  */
4080 static void fill_inode_item(struct btrfs_trans_handle *trans,
4081                             struct extent_buffer *leaf,
4082                             struct btrfs_inode_item *item,
4083                             struct inode *inode)
4084 {
4085         struct btrfs_map_token token;
4086
4087         btrfs_init_map_token(&token, leaf);
4088
4089         btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token);
4090         btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token);
4091         btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size,
4092                                    &token);
4093         btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token);
4094         btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token);
4095
4096         btrfs_set_token_timespec_sec(leaf, &item->atime,
4097                                      inode->i_atime.tv_sec, &token);
4098         btrfs_set_token_timespec_nsec(leaf, &item->atime,
4099                                       inode->i_atime.tv_nsec, &token);
4100
4101         btrfs_set_token_timespec_sec(leaf, &item->mtime,
4102                                      inode->i_mtime.tv_sec, &token);
4103         btrfs_set_token_timespec_nsec(leaf, &item->mtime,
4104                                       inode->i_mtime.tv_nsec, &token);
4105
4106         btrfs_set_token_timespec_sec(leaf, &item->ctime,
4107                                      inode->i_ctime.tv_sec, &token);
4108         btrfs_set_token_timespec_nsec(leaf, &item->ctime,
4109                                       inode->i_ctime.tv_nsec, &token);
4110
4111         btrfs_set_token_timespec_sec(leaf, &item->otime,
4112                                      BTRFS_I(inode)->i_otime.tv_sec, &token);
4113         btrfs_set_token_timespec_nsec(leaf, &item->otime,
4114                                       BTRFS_I(inode)->i_otime.tv_nsec, &token);
4115
4116         btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode),
4117                                      &token);
4118         btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation,
4119                                          &token);
4120         btrfs_set_token_inode_sequence(leaf, item, inode_peek_iversion(inode),
4121                                        &token);
4122         btrfs_set_token_inode_transid(leaf, item, trans->transid, &token);
4123         btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token);
4124         btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token);
4125         btrfs_set_token_inode_block_group(leaf, item, 0, &token);
4126 }
4127
4128 /*
4129  * copy everything in the in-memory inode into the btree.
4130  */
4131 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
4132                                 struct btrfs_root *root, struct inode *inode)
4133 {
4134         struct btrfs_inode_item *inode_item;
4135         struct btrfs_path *path;
4136         struct extent_buffer *leaf;
4137         int ret;
4138
4139         path = btrfs_alloc_path();
4140         if (!path)
4141                 return -ENOMEM;
4142
4143         path->leave_spinning = 1;
4144         ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location,
4145                                  1);
4146         if (ret) {
4147                 if (ret > 0)
4148                         ret = -ENOENT;
4149                 goto failed;
4150         }
4151
4152         leaf = path->nodes[0];
4153         inode_item = btrfs_item_ptr(leaf, path->slots[0],
4154                                     struct btrfs_inode_item);
4155
4156         fill_inode_item(trans, leaf, inode_item, inode);
4157         btrfs_mark_buffer_dirty(leaf);
4158         btrfs_set_inode_last_trans(trans, inode);
4159         ret = 0;
4160 failed:
4161         btrfs_free_path(path);
4162         return ret;
4163 }
4164
4165 /*
4166  * copy everything in the in-memory inode into the btree.
4167  */
4168 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans,
4169                                 struct btrfs_root *root, struct inode *inode)
4170 {
4171         struct btrfs_fs_info *fs_info = root->fs_info;
4172         int ret;
4173
4174         /*
4175          * If the inode is a free space inode, we can deadlock during commit
4176          * if we put it into the delayed code.
4177          *
4178          * The data relocation inode should also be directly updated
4179          * without delay
4180          */
4181         if (!btrfs_is_free_space_inode(BTRFS_I(inode))
4182             && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
4183             && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
4184                 btrfs_update_root_times(trans, root);
4185
4186                 ret = btrfs_delayed_update_inode(trans, root, inode);
4187                 if (!ret)
4188                         btrfs_set_inode_last_trans(trans, inode);
4189                 return ret;
4190         }
4191
4192         return btrfs_update_inode_item(trans, root, inode);
4193 }
4194
4195 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
4196                                          struct btrfs_root *root,
4197                                          struct inode *inode)
4198 {
4199         int ret;
4200
4201         ret = btrfs_update_inode(trans, root, inode);
4202         if (ret == -ENOSPC)
4203                 return btrfs_update_inode_item(trans, root, inode);
4204         return ret;
4205 }
4206
4207 /*
4208  * unlink helper that gets used here in inode.c and in the tree logging
4209  * recovery code.  It remove a link in a directory with a given name, and
4210  * also drops the back refs in the inode to the directory
4211  */
4212 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4213                                 struct btrfs_root *root,
4214                                 struct btrfs_inode *dir,
4215                                 struct btrfs_inode *inode,
4216                                 const char *name, int name_len)
4217 {
4218         struct btrfs_fs_info *fs_info = root->fs_info;
4219         struct btrfs_path *path;
4220         int ret = 0;
4221         struct btrfs_dir_item *di;
4222         u64 index;
4223         u64 ino = btrfs_ino(inode);
4224         u64 dir_ino = btrfs_ino(dir);
4225
4226         path = btrfs_alloc_path();
4227         if (!path) {
4228                 ret = -ENOMEM;
4229                 goto out;
4230         }
4231
4232         path->leave_spinning = 1;
4233         di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4234                                     name, name_len, -1);
4235         if (IS_ERR_OR_NULL(di)) {
4236                 ret = di ? PTR_ERR(di) : -ENOENT;
4237                 goto err;
4238         }
4239         ret = btrfs_delete_one_dir_name(trans, root, path, di);
4240         if (ret)
4241                 goto err;
4242         btrfs_release_path(path);
4243
4244         /*
4245          * If we don't have dir index, we have to get it by looking up
4246          * the inode ref, since we get the inode ref, remove it directly,
4247          * it is unnecessary to do delayed deletion.
4248          *
4249          * But if we have dir index, needn't search inode ref to get it.
4250          * Since the inode ref is close to the inode item, it is better
4251          * that we delay to delete it, and just do this deletion when
4252          * we update the inode item.
4253          */
4254         if (inode->dir_index) {
4255                 ret = btrfs_delayed_delete_inode_ref(inode);
4256                 if (!ret) {
4257                         index = inode->dir_index;
4258                         goto skip_backref;
4259                 }
4260         }
4261
4262         ret = btrfs_del_inode_ref(trans, root, name, name_len, ino,
4263                                   dir_ino, &index);
4264         if (ret) {
4265                 btrfs_info(fs_info,
4266                         "failed to delete reference to %.*s, inode %llu parent %llu",
4267                         name_len, name, ino, dir_ino);
4268                 btrfs_abort_transaction(trans, ret);
4269                 goto err;
4270         }
4271 skip_backref:
4272         ret = btrfs_delete_delayed_dir_index(trans, dir, index);
4273         if (ret) {
4274                 btrfs_abort_transaction(trans, ret);
4275                 goto err;
4276         }
4277
4278         ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode,
4279                         dir_ino);
4280         if (ret != 0 && ret != -ENOENT) {
4281                 btrfs_abort_transaction(trans, ret);
4282                 goto err;
4283         }
4284
4285         ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir,
4286                         index);
4287         if (ret == -ENOENT)
4288                 ret = 0;
4289         else if (ret)
4290                 btrfs_abort_transaction(trans, ret);
4291
4292         /*
4293          * If we have a pending delayed iput we could end up with the final iput
4294          * being run in btrfs-cleaner context.  If we have enough of these built
4295          * up we can end up burning a lot of time in btrfs-cleaner without any
4296          * way to throttle the unlinks.  Since we're currently holding a ref on
4297          * the inode we can run the delayed iput here without any issues as the
4298          * final iput won't be done until after we drop the ref we're currently
4299          * holding.
4300          */
4301         btrfs_run_delayed_iput(fs_info, inode);
4302 err:
4303         btrfs_free_path(path);
4304         if (ret)
4305                 goto out;
4306
4307         btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2);
4308         inode_inc_iversion(&inode->vfs_inode);
4309         inode_inc_iversion(&dir->vfs_inode);
4310         inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime =
4311                 dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode);
4312         ret = btrfs_update_inode(trans, root, &dir->vfs_inode);
4313 out:
4314         return ret;
4315 }
4316
4317 int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4318                        struct btrfs_root *root,
4319                        struct btrfs_inode *dir, struct btrfs_inode *inode,
4320                        const char *name, int name_len)
4321 {
4322         int ret;
4323         ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len);
4324         if (!ret) {
4325                 drop_nlink(&inode->vfs_inode);
4326                 ret = btrfs_update_inode(trans, root, &inode->vfs_inode);
4327         }
4328         return ret;
4329 }
4330
4331 /*
4332  * helper to start transaction for unlink and rmdir.
4333  *
4334  * unlink and rmdir are special in btrfs, they do not always free space, so
4335  * if we cannot make our reservations the normal way try and see if there is
4336  * plenty of slack room in the global reserve to migrate, otherwise we cannot
4337  * allow the unlink to occur.
4338  */
4339 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir)
4340 {
4341         struct btrfs_root *root = BTRFS_I(dir)->root;
4342
4343         /*
4344          * 1 for the possible orphan item
4345          * 1 for the dir item
4346          * 1 for the dir index
4347          * 1 for the inode ref
4348          * 1 for the inode
4349          */
4350         return btrfs_start_transaction_fallback_global_rsv(root, 5);
4351 }
4352
4353 static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
4354 {
4355         struct btrfs_root *root = BTRFS_I(dir)->root;
4356         struct btrfs_trans_handle *trans;
4357         struct inode *inode = d_inode(dentry);
4358         int ret;
4359
4360         trans = __unlink_start_trans(dir);
4361         if (IS_ERR(trans))
4362                 return PTR_ERR(trans);
4363
4364         btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4365                         0);
4366
4367         ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
4368                         BTRFS_I(d_inode(dentry)), dentry->d_name.name,
4369                         dentry->d_name.len);
4370         if (ret)
4371                 goto out;
4372
4373         if (inode->i_nlink == 0) {
4374                 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
4375                 if (ret)
4376                         goto out;
4377         }
4378
4379 out:
4380         btrfs_end_transaction(trans);
4381         btrfs_btree_balance_dirty(root->fs_info);
4382         return ret;
4383 }
4384
4385 static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
4386                                struct inode *dir, struct dentry *dentry)
4387 {
4388         struct btrfs_root *root = BTRFS_I(dir)->root;
4389         struct btrfs_inode *inode = BTRFS_I(d_inode(dentry));
4390         struct btrfs_path *path;
4391         struct extent_buffer *leaf;
4392         struct btrfs_dir_item *di;
4393         struct btrfs_key key;
4394         const char *name = dentry->d_name.name;
4395         int name_len = dentry->d_name.len;
4396         u64 index;
4397         int ret;
4398         u64 objectid;
4399         u64 dir_ino = btrfs_ino(BTRFS_I(dir));
4400
4401         if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) {
4402                 objectid = inode->root->root_key.objectid;
4403         } else if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {
4404                 objectid = inode->location.objectid;
4405         } else {
4406                 WARN_ON(1);
4407                 return -EINVAL;
4408         }
4409
4410         path = btrfs_alloc_path();
4411         if (!path)
4412                 return -ENOMEM;
4413
4414         di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4415                                    name, name_len, -1);
4416         if (IS_ERR_OR_NULL(di)) {
4417                 ret = di ? PTR_ERR(di) : -ENOENT;
4418                 goto out;
4419         }
4420
4421         leaf = path->nodes[0];
4422         btrfs_dir_item_key_to_cpu(leaf, di, &key);
4423         WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
4424         ret = btrfs_delete_one_dir_name(trans, root, path, di);
4425         if (ret) {
4426                 btrfs_abort_transaction(trans, ret);
4427                 goto out;
4428         }
4429         btrfs_release_path(path);
4430
4431         /*
4432          * This is a placeholder inode for a subvolume we didn't have a
4433          * reference to at the time of the snapshot creation.  In the meantime
4434          * we could have renamed the real subvol link into our snapshot, so
4435          * depending on btrfs_del_root_ref to return -ENOENT here is incorret.
4436          * Instead simply lookup the dir_index_item for this entry so we can
4437          * remove it.  Otherwise we know we have a ref to the root and we can
4438          * call btrfs_del_root_ref, and it _shouldn't_ fail.
4439          */
4440         if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {
4441                 di = btrfs_search_dir_index_item(root, path, dir_ino,
4442                                                  name, name_len);
4443                 if (IS_ERR_OR_NULL(di)) {
4444                         if (!di)
4445                                 ret = -ENOENT;
4446                         else
4447                                 ret = PTR_ERR(di);
4448                         btrfs_abort_transaction(trans, ret);
4449                         goto out;
4450                 }
4451
4452                 leaf = path->nodes[0];
4453                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4454                 index = key.offset;
4455                 btrfs_release_path(path);
4456         } else {
4457                 ret = btrfs_del_root_ref(trans, objectid,
4458                                          root->root_key.objectid, dir_ino,
4459                                          &index, name, name_len);
4460                 if (ret) {
4461                         btrfs_abort_transaction(trans, ret);
4462                         goto out;
4463                 }
4464         }
4465
4466         ret = btrfs_delete_delayed_dir_index(trans, BTRFS_I(dir), index);
4467         if (ret) {
4468                 btrfs_abort_transaction(trans, ret);
4469                 goto out;
4470         }
4471
4472         btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2);
4473         inode_inc_iversion(dir);
4474         dir->i_mtime = dir->i_ctime = current_time(dir);
4475         ret = btrfs_update_inode_fallback(trans, root, dir);
4476         if (ret)
4477                 btrfs_abort_transaction(trans, ret);
4478 out:
4479         btrfs_free_path(path);
4480         return ret;
4481 }
4482
4483 /*
4484  * Helper to check if the subvolume references other subvolumes or if it's
4485  * default.
4486  */
4487 static noinline int may_destroy_subvol(struct btrfs_root *root)
4488 {
4489         struct btrfs_fs_info *fs_info = root->fs_info;
4490         struct btrfs_path *path;
4491         struct btrfs_dir_item *di;
4492         struct btrfs_key key;
4493         u64 dir_id;
4494         int ret;
4495
4496         path = btrfs_alloc_path();
4497         if (!path)
4498                 return -ENOMEM;
4499
4500         /* Make sure this root isn't set as the default subvol */
4501         dir_id = btrfs_super_root_dir(fs_info->super_copy);
4502         di = btrfs_lookup_dir_item(NULL, fs_info->tree_root, path,
4503                                    dir_id, "default", 7, 0);
4504         if (di && !IS_ERR(di)) {
4505                 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);
4506                 if (key.objectid == root->root_key.objectid) {
4507                         ret = -EPERM;
4508                         btrfs_err(fs_info,
4509                                   "deleting default subvolume %llu is not allowed",
4510                                   key.objectid);
4511                         goto out;
4512                 }
4513                 btrfs_release_path(path);
4514         }
4515
4516         key.objectid = root->root_key.objectid;
4517         key.type = BTRFS_ROOT_REF_KEY;
4518         key.offset = (u64)-1;
4519
4520         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
4521         if (ret < 0)
4522                 goto out;
4523         BUG_ON(ret == 0);
4524
4525         ret = 0;
4526         if (path->slots[0] > 0) {
4527                 path->slots[0]--;
4528                 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
4529                 if (key.objectid == root->root_key.objectid &&
4530                     key.type == BTRFS_ROOT_REF_KEY)
4531                         ret = -ENOTEMPTY;
4532         }
4533 out:
4534         btrfs_free_path(path);
4535         return ret;
4536 }
4537
4538 /* Delete all dentries for inodes belonging to the root */
4539 static void btrfs_prune_dentries(struct btrfs_root *root)
4540 {
4541         struct btrfs_fs_info *fs_info = root->fs_info;
4542         struct rb_node *node;
4543         struct rb_node *prev;
4544         struct btrfs_inode *entry;
4545         struct inode *inode;
4546         u64 objectid = 0;
4547
4548         if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
4549                 WARN_ON(btrfs_root_refs(&root->root_item) != 0);
4550
4551         spin_lock(&root->inode_lock);
4552 again:
4553         node = root->inode_tree.rb_node;
4554         prev = NULL;
4555         while (node) {
4556                 prev = node;
4557                 entry = rb_entry(node, struct btrfs_inode, rb_node);
4558
4559                 if (objectid < btrfs_ino(entry))
4560                         node = node->rb_left;
4561                 else if (objectid > btrfs_ino(entry))
4562                         node = node->rb_right;
4563                 else
4564                         break;
4565         }
4566         if (!node) {
4567                 while (prev) {
4568                         entry = rb_entry(prev, struct btrfs_inode, rb_node);
4569                         if (objectid <= btrfs_ino(entry)) {
4570                                 node = prev;
4571                                 break;
4572                         }
4573                         prev = rb_next(prev);
4574                 }
4575         }
4576         while (node) {
4577                 entry = rb_entry(node, struct btrfs_inode, rb_node);
4578                 objectid = btrfs_ino(entry) + 1;
4579                 inode = igrab(&entry->vfs_inode);
4580                 if (inode) {
4581                         spin_unlock(&root->inode_lock);
4582                         if (atomic_read(&inode->i_count) > 1)
4583                                 d_prune_aliases(inode);
4584                         /*
4585                          * btrfs_drop_inode will have it removed from the inode
4586                          * cache when its usage count hits zero.
4587                          */
4588                         iput(inode);
4589                         cond_resched();
4590                         spin_lock(&root->inode_lock);
4591                         goto again;
4592                 }
4593
4594                 if (cond_resched_lock(&root->inode_lock))
4595                         goto again;
4596
4597                 node = rb_next(node);
4598         }
4599         spin_unlock(&root->inode_lock);
4600 }
4601
4602 int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry)
4603 {
4604         struct btrfs_fs_info *fs_info = btrfs_sb(dentry->d_sb);
4605         struct btrfs_root *root = BTRFS_I(dir)->root;
4606         struct inode *inode = d_inode(dentry);
4607         struct btrfs_root *dest = BTRFS_I(inode)->root;
4608         struct btrfs_trans_handle *trans;
4609         struct btrfs_block_rsv block_rsv;
4610         u64 root_flags;
4611         int ret;
4612         int err;
4613
4614         /*
4615          * Don't allow to delete a subvolume with send in progress. This is
4616          * inside the inode lock so the error handling that has to drop the bit
4617          * again is not run concurrently.
4618          */
4619         spin_lock(&dest->root_item_lock);
4620         if (dest->send_in_progress) {
4621                 spin_unlock(&dest->root_item_lock);
4622                 btrfs_warn(fs_info,
4623                            "attempt to delete subvolume %llu during send",
4624                            dest->root_key.objectid);
4625                 return -EPERM;
4626         }
4627         root_flags = btrfs_root_flags(&dest->root_item);
4628         btrfs_set_root_flags(&dest->root_item,
4629                              root_flags | BTRFS_ROOT_SUBVOL_DEAD);
4630         spin_unlock(&dest->root_item_lock);
4631
4632         down_write(&fs_info->subvol_sem);
4633
4634         err = may_destroy_subvol(dest);
4635         if (err)
4636                 goto out_up_write;
4637
4638         btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP);
4639         /*
4640          * One for dir inode,
4641          * two for dir entries,
4642          * two for root ref/backref.
4643          */
4644         err = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true);
4645         if (err)
4646                 goto out_up_write;
4647
4648         trans = btrfs_start_transaction(root, 0);
4649         if (IS_ERR(trans)) {
4650                 err = PTR_ERR(trans);
4651                 goto out_release;
4652         }
4653         trans->block_rsv = &block_rsv;
4654         trans->bytes_reserved = block_rsv.size;
4655
4656         btrfs_record_snapshot_destroy(trans, BTRFS_I(dir));
4657
4658         ret = btrfs_unlink_subvol(trans, dir, dentry);
4659         if (ret) {
4660                 err = ret;
4661                 btrfs_abort_transaction(trans, ret);
4662                 goto out_end_trans;
4663         }
4664
4665         btrfs_record_root_in_trans(trans, dest);
4666
4667         memset(&dest->root_item.drop_progress, 0,
4668                 sizeof(dest->root_item.drop_progress));
4669         dest->root_item.drop_level = 0;
4670         btrfs_set_root_refs(&dest->root_item, 0);
4671
4672         if (!test_and_set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &dest->state)) {
4673                 ret = btrfs_insert_orphan_item(trans,
4674                                         fs_info->tree_root,
4675                                         dest->root_key.objectid);
4676                 if (ret) {
4677                         btrfs_abort_transaction(trans, ret);
4678                         err = ret;
4679                         goto out_end_trans;
4680                 }
4681         }
4682
4683         ret = btrfs_uuid_tree_remove(trans, dest->root_item.uuid,
4684                                   BTRFS_UUID_KEY_SUBVOL,
4685                                   dest->root_key.objectid);
4686         if (ret && ret != -ENOENT) {
4687                 btrfs_abort_transaction(trans, ret);
4688                 err = ret;
4689                 goto out_end_trans;
4690         }
4691         if (!btrfs_is_empty_uuid(dest->root_item.received_uuid)) {
4692                 ret = btrfs_uuid_tree_remove(trans,
4693                                           dest->root_item.received_uuid,
4694                                           BTRFS_UUID_KEY_RECEIVED_SUBVOL,
4695                                           dest->root_key.objectid);
4696                 if (ret && ret != -ENOENT) {
4697                         btrfs_abort_transaction(trans, ret);
4698                         err = ret;
4699                         goto out_end_trans;
4700                 }
4701         }
4702
4703         free_anon_bdev(dest->anon_dev);
4704         dest->anon_dev = 0;
4705 out_end_trans:
4706         trans->block_rsv = NULL;
4707         trans->bytes_reserved = 0;
4708         ret = btrfs_end_transaction(trans);
4709         if (ret && !err)
4710                 err = ret;
4711         inode->i_flags |= S_DEAD;
4712 out_release:
4713         btrfs_subvolume_release_metadata(fs_info, &block_rsv);
4714 out_up_write:
4715         up_write(&fs_info->subvol_sem);
4716         if (err) {
4717                 spin_lock(&dest->root_item_lock);
4718                 root_flags = btrfs_root_flags(&dest->root_item);
4719                 btrfs_set_root_flags(&dest->root_item,
4720                                 root_flags & ~BTRFS_ROOT_SUBVOL_DEAD);
4721                 spin_unlock(&dest->root_item_lock);
4722         } else {
4723                 d_invalidate(dentry);
4724                 btrfs_prune_dentries(dest);
4725                 ASSERT(dest->send_in_progress == 0);
4726
4727                 /* the last ref */
4728                 if (dest->ino_cache_inode) {
4729                         iput(dest->ino_cache_inode);
4730                         dest->ino_cache_inode = NULL;
4731                 }
4732         }
4733
4734         return err;
4735 }
4736
4737 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
4738 {
4739         struct inode *inode = d_inode(dentry);
4740         int err = 0;
4741         struct btrfs_root *root = BTRFS_I(dir)->root;
4742         struct btrfs_trans_handle *trans;
4743         u64 last_unlink_trans;
4744
4745         if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
4746                 return -ENOTEMPTY;
4747         if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID)
4748                 return btrfs_delete_subvolume(dir, dentry);
4749
4750         trans = __unlink_start_trans(dir);
4751         if (IS_ERR(trans))
4752                 return PTR_ERR(trans);
4753
4754         if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
4755                 err = btrfs_unlink_subvol(trans, dir, dentry);
4756                 goto out;
4757         }
4758
4759         err = btrfs_orphan_add(trans, BTRFS_I(inode));
4760         if (err)
4761                 goto out;
4762
4763         last_unlink_trans = BTRFS_I(inode)->last_unlink_trans;
4764
4765         /* now the directory is empty */
4766         err = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
4767                         BTRFS_I(d_inode(dentry)), dentry->d_name.name,
4768                         dentry->d_name.len);
4769         if (!err) {
4770                 btrfs_i_size_write(BTRFS_I(inode), 0);
4771                 /*
4772                  * Propagate the last_unlink_trans value of the deleted dir to
4773                  * its parent directory. This is to prevent an unrecoverable
4774                  * log tree in the case we do something like this:
4775                  * 1) create dir foo
4776                  * 2) create snapshot under dir foo
4777                  * 3) delete the snapshot
4778                  * 4) rmdir foo
4779                  * 5) mkdir foo
4780                  * 6) fsync foo or some file inside foo
4781                  */
4782                 if (last_unlink_trans >= trans->transid)
4783                         BTRFS_I(dir)->last_unlink_trans = last_unlink_trans;
4784         }
4785 out:
4786         btrfs_end_transaction(trans);
4787         btrfs_btree_balance_dirty(root->fs_info);
4788
4789         return err;
4790 }
4791
4792 /*
4793  * Return this if we need to call truncate_block for the last bit of the
4794  * truncate.
4795  */
4796 #define NEED_TRUNCATE_BLOCK 1
4797
4798 /*
4799  * this can truncate away extent items, csum items and directory items.
4800  * It starts at a high offset and removes keys until it can't find
4801  * any higher than new_size
4802  *
4803  * csum items that cross the new i_size are truncated to the new size
4804  * as well.
4805  *
4806  * min_type is the minimum key type to truncate down to.  If set to 0, this
4807  * will kill all the items on this inode, including the INODE_ITEM_KEY.
4808  */
4809 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
4810                                struct btrfs_root *root,
4811                                struct inode *inode,
4812                                u64 new_size, u32 min_type)
4813 {
4814         struct btrfs_fs_info *fs_info = root->fs_info;
4815         struct btrfs_path *path;
4816         struct extent_buffer *leaf;
4817         struct btrfs_file_extent_item *fi;
4818         struct btrfs_key key;
4819         struct btrfs_key found_key;
4820         u64 extent_start = 0;
4821         u64 extent_num_bytes = 0;
4822         u64 extent_offset = 0;
4823         u64 item_end = 0;
4824         u64 last_size = new_size;
4825         u32 found_type = (u8)-1;
4826         int found_extent;
4827         int del_item;
4828         int pending_del_nr = 0;
4829         int pending_del_slot = 0;
4830         int extent_type = -1;
4831         int ret;
4832         u64 ino = btrfs_ino(BTRFS_I(inode));
4833         u64 bytes_deleted = 0;
4834         bool be_nice = false;
4835         bool should_throttle = false;
4836         const u64 lock_start = ALIGN_DOWN(new_size, fs_info->sectorsize);
4837         struct extent_state *cached_state = NULL;
4838
4839         BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY);
4840
4841         /*
4842          * for non-free space inodes and ref cows, we want to back off from
4843          * time to time
4844          */
4845         if (!btrfs_is_free_space_inode(BTRFS_I(inode)) &&
4846             test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4847                 be_nice = true;
4848
4849         path = btrfs_alloc_path();
4850         if (!path)
4851                 return -ENOMEM;
4852         path->reada = READA_BACK;
4853
4854         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID)
4855                 lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, (u64)-1,
4856                                  &cached_state);
4857
4858         /*
4859          * We want to drop from the next block forward in case this new size is
4860          * not block aligned since we will be keeping the last block of the
4861          * extent just the way it is.
4862          */
4863         if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4864             root == fs_info->tree_root)
4865                 btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size,
4866                                         fs_info->sectorsize),
4867                                         (u64)-1, 0);
4868
4869         /*
4870          * This function is also used to drop the items in the log tree before
4871          * we relog the inode, so if root != BTRFS_I(inode)->root, it means
4872          * it is used to drop the logged items. So we shouldn't kill the delayed
4873          * items.
4874          */
4875         if (min_type == 0 && root == BTRFS_I(inode)->root)
4876                 btrfs_kill_delayed_inode_items(BTRFS_I(inode));
4877
4878         key.objectid = ino;
4879         key.offset = (u64)-1;
4880         key.type = (u8)-1;
4881
4882 search_again:
4883         /*
4884          * with a 16K leaf size and 128MB extents, you can actually queue
4885          * up a huge file in a single leaf.  Most of the time that
4886          * bytes_deleted is > 0, it will be huge by the time we get here
4887          */
4888         if (be_nice && bytes_deleted > SZ_32M &&
4889             btrfs_should_end_transaction(trans)) {
4890                 ret = -EAGAIN;
4891                 goto out;
4892         }
4893
4894         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
4895         if (ret < 0)
4896                 goto out;
4897
4898         if (ret > 0) {
4899                 ret = 0;
4900                 /* there are no items in the tree for us to truncate, we're
4901                  * done
4902                  */
4903                 if (path->slots[0] == 0)
4904                         goto out;
4905                 path->slots[0]--;
4906         }
4907
4908         while (1) {
4909                 fi = NULL;
4910                 leaf = path->nodes[0];
4911                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
4912                 found_type = found_key.type;
4913
4914                 if (found_key.objectid != ino)
4915                         break;
4916
4917                 if (found_type < min_type)
4918                         break;
4919
4920                 item_end = found_key.offset;
4921                 if (found_type == BTRFS_EXTENT_DATA_KEY) {
4922                         fi = btrfs_item_ptr(leaf, path->slots[0],
4923                                             struct btrfs_file_extent_item);
4924                         extent_type = btrfs_file_extent_type(leaf, fi);
4925                         if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4926                                 item_end +=
4927                                     btrfs_file_extent_num_bytes(leaf, fi);
4928
4929                                 trace_btrfs_truncate_show_fi_regular(
4930                                         BTRFS_I(inode), leaf, fi,
4931                                         found_key.offset);
4932                         } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4933                                 item_end += btrfs_file_extent_ram_bytes(leaf,
4934                                                                         fi);
4935
4936                                 trace_btrfs_truncate_show_fi_inline(
4937                                         BTRFS_I(inode), leaf, fi, path->slots[0],
4938                                         found_key.offset);
4939                         }
4940                         item_end--;
4941                 }
4942                 if (found_type > min_type) {
4943                         del_item = 1;
4944                 } else {
4945                         if (item_end < new_size)
4946                                 break;
4947                         if (found_key.offset >= new_size)
4948                                 del_item = 1;
4949                         else
4950                                 del_item = 0;
4951                 }
4952                 found_extent = 0;
4953                 /* FIXME, shrink the extent if the ref count is only 1 */
4954                 if (found_type != BTRFS_EXTENT_DATA_KEY)
4955                         goto delete;
4956
4957                 if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4958                         u64 num_dec;
4959                         extent_start = btrfs_file_extent_disk_bytenr(leaf, fi);
4960                         if (!del_item) {
4961                                 u64 orig_num_bytes =
4962                                         btrfs_file_extent_num_bytes(leaf, fi);
4963                                 extent_num_bytes = ALIGN(new_size -
4964                                                 found_key.offset,
4965                                                 fs_info->sectorsize);
4966                                 btrfs_set_file_extent_num_bytes(leaf, fi,
4967                                                          extent_num_bytes);
4968                                 num_dec = (orig_num_bytes -
4969                                            extent_num_bytes);
4970                                 if (test_bit(BTRFS_ROOT_REF_COWS,
4971                                              &root->state) &&
4972                                     extent_start != 0)
4973                                         inode_sub_bytes(inode, num_dec);
4974                                 btrfs_mark_buffer_dirty(leaf);
4975                         } else {
4976                                 extent_num_bytes =
4977                                         btrfs_file_extent_disk_num_bytes(leaf,
4978                                                                          fi);
4979                                 extent_offset = found_key.offset -
4980                                         btrfs_file_extent_offset(leaf, fi);
4981
4982                                 /* FIXME blocksize != 4096 */
4983                                 num_dec = btrfs_file_extent_num_bytes(leaf, fi);
4984                                 if (extent_start != 0) {
4985                                         found_extent = 1;
4986                                         if (test_bit(BTRFS_ROOT_REF_COWS,
4987                                                      &root->state))
4988                                                 inode_sub_bytes(inode, num_dec);
4989                                 }
4990                         }
4991                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4992                         /*
4993                          * we can't truncate inline items that have had
4994                          * special encodings
4995                          */
4996                         if (!del_item &&
4997                             btrfs_file_extent_encryption(leaf, fi) == 0 &&
4998                             btrfs_file_extent_other_encoding(leaf, fi) == 0 &&
4999                             btrfs_file_extent_compression(leaf, fi) == 0) {
5000                                 u32 size = (u32)(new_size - found_key.offset);
5001
5002                                 btrfs_set_file_extent_ram_bytes(leaf, fi, size);
5003                                 size = btrfs_file_extent_calc_inline_size(size);
5004                                 btrfs_truncate_item(path, size, 1);
5005                         } else if (!del_item) {
5006                                 /*
5007                                  * We have to bail so the last_size is set to
5008                                  * just before this extent.
5009                                  */
5010                                 ret = NEED_TRUNCATE_BLOCK;
5011                                 break;
5012                         }
5013
5014                         if (test_bit(BTRFS_ROOT_REF_COWS, &root->state))
5015                                 inode_sub_bytes(inode, item_end + 1 - new_size);
5016                 }
5017 delete:
5018                 if (del_item)
5019                         last_size = found_key.offset;
5020                 else
5021                         last_size = new_size;
5022                 if (del_item) {
5023                         if (!pending_del_nr) {
5024                                 /* no pending yet, add ourselves */
5025                                 pending_del_slot = path->slots[0];
5026                                 pending_del_nr = 1;
5027                         } else if (pending_del_nr &&
5028                                    path->slots[0] + 1 == pending_del_slot) {
5029                                 /* hop on the pending chunk */
5030                                 pending_del_nr++;
5031                                 pending_del_slot = path->slots[0];
5032                         } else {
5033                                 BUG();
5034                         }
5035                 } else {
5036                         break;
5037                 }
5038                 should_throttle = false;
5039
5040                 if (found_extent &&
5041                     (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
5042                      root == fs_info->tree_root)) {
5043                         struct btrfs_ref ref = { 0 };
5044
5045                         bytes_deleted += extent_num_bytes;
5046
5047                         btrfs_init_generic_ref(&ref, BTRFS_DROP_DELAYED_REF,
5048                                         extent_start, extent_num_bytes, 0);
5049                         ref.real_root = root->root_key.objectid;
5050                         btrfs_init_data_ref(&ref, btrfs_header_owner(leaf),
5051                                         ino, extent_offset);
5052                         ret = btrfs_free_extent(trans, &ref);
5053                         if (ret) {
5054                                 btrfs_abort_transaction(trans, ret);
5055                                 break;
5056                         }
5057                         if (be_nice) {
5058                                 if (btrfs_should_throttle_delayed_refs(trans))
5059                                         should_throttle = true;
5060                         }
5061                 }
5062
5063                 if (found_type == BTRFS_INODE_ITEM_KEY)
5064                         break;
5065
5066                 if (path->slots[0] == 0 ||
5067                     path->slots[0] != pending_del_slot ||
5068                     should_throttle) {
5069                         if (pending_del_nr) {
5070                                 ret = btrfs_del_items(trans, root, path,
5071                                                 pending_del_slot,
5072                                                 pending_del_nr);
5073                                 if (ret) {
5074                                         btrfs_abort_transaction(trans, ret);
5075                                         break;
5076                                 }
5077                                 pending_del_nr = 0;
5078                         }
5079                         btrfs_release_path(path);
5080
5081                         /*
5082                          * We can generate a lot of delayed refs, so we need to
5083                          * throttle every once and a while and make sure we're
5084                          * adding enough space to keep up with the work we are
5085                          * generating.  Since we hold a transaction here we
5086                          * can't flush, and we don't want to FLUSH_LIMIT because
5087                          * we could have generated too many delayed refs to
5088                          * actually allocate, so just bail if we're short and
5089                          * let the normal reservation dance happen higher up.
5090                          */
5091                         if (should_throttle) {
5092                                 ret = btrfs_delayed_refs_rsv_refill(fs_info,
5093                                                         BTRFS_RESERVE_NO_FLUSH);
5094                                 if (ret) {
5095                                         ret = -EAGAIN;
5096                                         break;
5097                                 }
5098                         }
5099                         goto search_again;
5100                 } else {
5101                         path->slots[0]--;
5102                 }
5103         }
5104 out:
5105         if (ret >= 0 && pending_del_nr) {
5106                 int err;
5107
5108                 err = btrfs_del_items(trans, root, path, pending_del_slot,
5109                                       pending_del_nr);
5110                 if (err) {
5111                         btrfs_abort_transaction(trans, err);
5112                         ret = err;
5113                 }
5114         }
5115         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
5116                 ASSERT(last_size >= new_size);
5117                 if (!ret && last_size > new_size)
5118                         last_size = new_size;
5119                 btrfs_ordered_update_i_size(inode, last_size, NULL);
5120                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start,
5121                                      (u64)-1, &cached_state);
5122         }
5123
5124         btrfs_free_path(path);
5125         return ret;
5126 }
5127
5128 /*
5129  * btrfs_truncate_block - read, zero a chunk and write a block
5130  * @inode - inode that we're zeroing
5131  * @from - the offset to start zeroing
5132  * @len - the length to zero, 0 to zero the entire range respective to the
5133  *      offset
5134  * @front - zero up to the offset instead of from the offset on
5135  *
5136  * This will find the block for the "from" offset and cow the block and zero the
5137  * part we want to zero.  This is used with truncate and hole punching.
5138  */
5139 int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len,
5140                         int front)
5141 {
5142         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5143         struct address_space *mapping = inode->i_mapping;
5144         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
5145         struct btrfs_ordered_extent *ordered;
5146         struct extent_state *cached_state = NULL;
5147         struct extent_changeset *data_reserved = NULL;
5148         char *kaddr;
5149         bool only_release_metadata = false;
5150         u32 blocksize = fs_info->sectorsize;
5151         pgoff_t index = from >> PAGE_SHIFT;
5152         unsigned offset = from & (blocksize - 1);
5153         struct page *page;
5154         gfp_t mask = btrfs_alloc_write_mask(mapping);
5155         size_t write_bytes = blocksize;
5156         int ret = 0;
5157         u64 block_start;
5158         u64 block_end;
5159
5160         if (IS_ALIGNED(offset, blocksize) &&
5161             (!len || IS_ALIGNED(len, blocksize)))
5162                 goto out;
5163
5164         block_start = round_down(from, blocksize);
5165         block_end = block_start + blocksize - 1;
5166
5167
5168         ret = btrfs_check_data_free_space(inode, &data_reserved, block_start,
5169                                           blocksize);
5170         if (ret < 0) {
5171                 if ((BTRFS_I(inode)->flags & (BTRFS_INODE_NODATACOW |
5172                                               BTRFS_INODE_PREALLOC)) &&
5173                     btrfs_check_can_nocow(BTRFS_I(inode), block_start,
5174                                           &write_bytes) > 0) {
5175                         /* For nocow case, no need to reserve data space */
5176                         only_release_metadata = true;
5177                 } else {
5178                         goto out;
5179                 }
5180         }
5181         ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), blocksize);
5182         if (ret < 0) {
5183                 if (!only_release_metadata)
5184                         btrfs_free_reserved_data_space(inode, data_reserved,
5185                                         block_start, blocksize);
5186                 goto out;
5187         }
5188 again:
5189         page = find_or_create_page(mapping, index, mask);
5190         if (!page) {
5191                 btrfs_delalloc_release_space(inode, data_reserved,
5192                                              block_start, blocksize, true);
5193                 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize);
5194                 ret = -ENOMEM;
5195                 goto out;
5196         }
5197
5198         if (!PageUptodate(page)) {
5199                 ret = btrfs_readpage(NULL, page);
5200                 lock_page(page);
5201                 if (page->mapping != mapping) {
5202                         unlock_page(page);
5203                         put_page(page);
5204                         goto again;
5205                 }
5206                 if (!PageUptodate(page)) {
5207                         ret = -EIO;
5208                         goto out_unlock;
5209                 }
5210         }
5211         wait_on_page_writeback(page);
5212
5213         lock_extent_bits(io_tree, block_start, block_end, &cached_state);
5214         set_page_extent_mapped(page);
5215
5216         ordered = btrfs_lookup_ordered_extent(inode, block_start);
5217         if (ordered) {
5218                 unlock_extent_cached(io_tree, block_start, block_end,
5219                                      &cached_state);
5220                 unlock_page(page);
5221                 put_page(page);
5222                 btrfs_start_ordered_extent(inode, ordered, 1);
5223                 btrfs_put_ordered_extent(ordered);
5224                 goto again;
5225         }
5226
5227         clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end,
5228                          EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
5229                          0, 0, &cached_state);
5230
5231         ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0,
5232                                         &cached_state);
5233         if (ret) {
5234                 unlock_extent_cached(io_tree, block_start, block_end,
5235                                      &cached_state);
5236                 goto out_unlock;
5237         }
5238
5239         if (offset != blocksize) {
5240                 if (!len)
5241                         len = blocksize - offset;
5242                 kaddr = kmap(page);
5243                 if (front)
5244                         memset(kaddr + (block_start - page_offset(page)),
5245                                 0, offset);
5246                 else
5247                         memset(kaddr + (block_start - page_offset(page)) +  offset,
5248                                 0, len);
5249                 flush_dcache_page(page);
5250                 kunmap(page);
5251         }
5252         ClearPageChecked(page);
5253         set_page_dirty(page);
5254         unlock_extent_cached(io_tree, block_start, block_end, &cached_state);
5255
5256         if (only_release_metadata)
5257                 set_extent_bit(&BTRFS_I(inode)->io_tree, block_start,
5258                                 block_end, EXTENT_NORESERVE, NULL, NULL,
5259                                 GFP_NOFS);
5260
5261 out_unlock:
5262         if (ret) {
5263                 if (only_release_metadata)
5264                         btrfs_delalloc_release_metadata(BTRFS_I(inode),
5265                                         blocksize, true);
5266                 else
5267                         btrfs_delalloc_release_space(inode, data_reserved,
5268                                         block_start, blocksize, true);
5269         }
5270         btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize);
5271         unlock_page(page);
5272         put_page(page);
5273 out:
5274         if (only_release_metadata)
5275                 btrfs_end_write_no_snapshotting(BTRFS_I(inode)->root);
5276         extent_changeset_free(data_reserved);
5277         return ret;
5278 }
5279
5280 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode,
5281                              u64 offset, u64 len)
5282 {
5283         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5284         struct btrfs_trans_handle *trans;
5285         int ret;
5286
5287         /*
5288          * Still need to make sure the inode looks like it's been updated so
5289          * that any holes get logged if we fsync.
5290          */
5291         if (btrfs_fs_incompat(fs_info, NO_HOLES)) {
5292                 BTRFS_I(inode)->last_trans = fs_info->generation;
5293                 BTRFS_I(inode)->last_sub_trans = root->log_transid;
5294                 BTRFS_I(inode)->last_log_commit = root->last_log_commit;
5295                 return 0;
5296         }
5297
5298         /*
5299          * 1 - for the one we're dropping
5300          * 1 - for the one we're adding
5301          * 1 - for updating the inode.
5302          */
5303         trans = btrfs_start_transaction(root, 3);
5304         if (IS_ERR(trans))
5305                 return PTR_ERR(trans);
5306
5307         ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1);
5308         if (ret) {
5309                 btrfs_abort_transaction(trans, ret);
5310                 btrfs_end_transaction(trans);
5311                 return ret;
5312         }
5313
5314         ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)),
5315                         offset, 0, 0, len, 0, len, 0, 0, 0);
5316         if (ret)
5317                 btrfs_abort_transaction(trans, ret);
5318         else
5319                 btrfs_update_inode(trans, root, inode);
5320         btrfs_end_transaction(trans);
5321         return ret;
5322 }
5323
5324 /*
5325  * This function puts in dummy file extents for the area we're creating a hole
5326  * for.  So if we are truncating this file to a larger size we need to insert
5327  * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
5328  * the range between oldsize and size
5329  */
5330 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size)
5331 {
5332         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5333         struct btrfs_root *root = BTRFS_I(inode)->root;
5334         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
5335         struct extent_map *em = NULL;
5336         struct extent_state *cached_state = NULL;
5337         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
5338         u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);
5339         u64 block_end = ALIGN(size, fs_info->sectorsize);
5340         u64 last_byte;
5341         u64 cur_offset;
5342         u64 hole_size;
5343         int err = 0;
5344
5345         /*
5346          * If our size started in the middle of a block we need to zero out the
5347          * rest of the block before we expand the i_size, otherwise we could
5348          * expose stale data.
5349          */
5350         err = btrfs_truncate_block(inode, oldsize, 0, 0);
5351         if (err)
5352                 return err;
5353
5354         if (size <= hole_start)
5355                 return 0;
5356
5357         btrfs_lock_and_flush_ordered_range(io_tree, BTRFS_I(inode), hole_start,
5358                                            block_end - 1, &cached_state);
5359         cur_offset = hole_start;
5360         while (1) {
5361                 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
5362                                 block_end - cur_offset, 0);
5363                 if (IS_ERR(em)) {
5364                         err = PTR_ERR(em);
5365                         em = NULL;
5366                         break;
5367                 }
5368                 last_byte = min(extent_map_end(em), block_end);
5369                 last_byte = ALIGN(last_byte, fs_info->sectorsize);
5370                 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
5371                         struct extent_map *hole_em;
5372                         hole_size = last_byte - cur_offset;
5373
5374                         err = maybe_insert_hole(root, inode, cur_offset,
5375                                                 hole_size);
5376                         if (err)
5377                                 break;
5378                         btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
5379                                                 cur_offset + hole_size - 1, 0);
5380                         hole_em = alloc_extent_map();
5381                         if (!hole_em) {
5382                                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
5383                                         &BTRFS_I(inode)->runtime_flags);
5384                                 goto next;
5385                         }
5386                         hole_em->start = cur_offset;
5387                         hole_em->len = hole_size;
5388                         hole_em->orig_start = cur_offset;
5389
5390                         hole_em->block_start = EXTENT_MAP_HOLE;
5391                         hole_em->block_len = 0;
5392                         hole_em->orig_block_len = 0;
5393                         hole_em->ram_bytes = hole_size;
5394                         hole_em->bdev = fs_info->fs_devices->latest_bdev;
5395                         hole_em->compress_type = BTRFS_COMPRESS_NONE;
5396                         hole_em->generation = fs_info->generation;
5397
5398                         while (1) {
5399                                 write_lock(&em_tree->lock);
5400                                 err = add_extent_mapping(em_tree, hole_em, 1);
5401                                 write_unlock(&em_tree->lock);
5402                                 if (err != -EEXIST)
5403                                         break;
5404                                 btrfs_drop_extent_cache(BTRFS_I(inode),
5405                                                         cur_offset,
5406                                                         cur_offset +
5407                                                         hole_size - 1, 0);
5408                         }
5409                         free_extent_map(hole_em);
5410                 }
5411 next:
5412                 free_extent_map(em);
5413                 em = NULL;
5414                 cur_offset = last_byte;
5415                 if (cur_offset >= block_end)
5416                         break;
5417         }
5418         free_extent_map(em);
5419         unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state);
5420         return err;
5421 }
5422
5423 static int btrfs_setsize(struct inode *inode, struct iattr *attr)
5424 {
5425         struct btrfs_root *root = BTRFS_I(inode)->root;
5426         struct btrfs_trans_handle *trans;
5427         loff_t oldsize = i_size_read(inode);
5428         loff_t newsize = attr->ia_size;
5429         int mask = attr->ia_valid;
5430         int ret;
5431
5432         /*
5433          * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
5434          * special case where we need to update the times despite not having
5435          * these flags set.  For all other operations the VFS set these flags
5436          * explicitly if it wants a timestamp update.
5437          */
5438         if (newsize != oldsize) {
5439                 inode_inc_iversion(inode);
5440                 if (!(mask & (ATTR_CTIME | ATTR_MTIME)))
5441                         inode->i_ctime = inode->i_mtime =
5442                                 current_time(inode);
5443         }
5444
5445         if (newsize > oldsize) {
5446                 /*
5447                  * Don't do an expanding truncate while snapshotting is ongoing.
5448                  * This is to ensure the snapshot captures a fully consistent
5449                  * state of this file - if the snapshot captures this expanding
5450                  * truncation, it must capture all writes that happened before
5451                  * this truncation.
5452                  */
5453                 btrfs_wait_for_snapshot_creation(root);
5454                 ret = btrfs_cont_expand(inode, oldsize, newsize);
5455                 if (ret) {
5456                         btrfs_end_write_no_snapshotting(root);
5457                         return ret;
5458                 }
5459
5460                 trans = btrfs_start_transaction(root, 1);
5461                 if (IS_ERR(trans)) {
5462                         btrfs_end_write_no_snapshotting(root);
5463                         return PTR_ERR(trans);
5464                 }
5465
5466                 i_size_write(inode, newsize);
5467                 btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL);
5468                 pagecache_isize_extended(inode, oldsize, newsize);
5469                 ret = btrfs_update_inode(trans, root, inode);
5470                 btrfs_end_write_no_snapshotting(root);
5471                 btrfs_end_transaction(trans);
5472         } else {
5473
5474                 /*
5475                  * We're truncating a file that used to have good data down to
5476                  * zero. Make sure it gets into the ordered flush list so that
5477                  * any new writes get down to disk quickly.
5478                  */
5479                 if (newsize == 0)
5480                         set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
5481                                 &BTRFS_I(inode)->runtime_flags);
5482
5483                 truncate_setsize(inode, newsize);
5484
5485                 /* Disable nonlocked read DIO to avoid the endless truncate */
5486                 btrfs_inode_block_unlocked_dio(BTRFS_I(inode));
5487                 inode_dio_wait(inode);
5488                 btrfs_inode_resume_unlocked_dio(BTRFS_I(inode));
5489
5490                 ret = btrfs_truncate(inode, newsize == oldsize);
5491                 if (ret && inode->i_nlink) {
5492                         int err;
5493
5494                         /*
5495                          * Truncate failed, so fix up the in-memory size. We
5496                          * adjusted disk_i_size down as we removed extents, so
5497                          * wait for disk_i_size to be stable and then update the
5498                          * in-memory size to match.
5499                          */
5500                         err = btrfs_wait_ordered_range(inode, 0, (u64)-1);
5501                         if (err)
5502                                 return err;
5503                         i_size_write(inode, BTRFS_I(inode)->disk_i_size);
5504                 }
5505         }
5506
5507         return ret;
5508 }
5509
5510 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr)
5511 {
5512         struct inode *inode = d_inode(dentry);
5513         struct btrfs_root *root = BTRFS_I(inode)->root;
5514         int err;
5515
5516         if (btrfs_root_readonly(root))
5517                 return -EROFS;
5518
5519         err = setattr_prepare(dentry, attr);
5520         if (err)
5521                 return err;
5522
5523         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
5524                 err = btrfs_setsize(inode, attr);
5525                 if (err)
5526                         return err;
5527         }
5528
5529         if (attr->ia_valid) {
5530                 setattr_copy(inode, attr);
5531                 inode_inc_iversion(inode);
5532                 err = btrfs_dirty_inode(inode);
5533
5534                 if (!err && attr->ia_valid & ATTR_MODE)
5535                         err = posix_acl_chmod(inode, inode->i_mode);
5536         }
5537
5538         return err;
5539 }
5540
5541 /*
5542  * While truncating the inode pages during eviction, we get the VFS calling
5543  * btrfs_invalidatepage() against each page of the inode. This is slow because
5544  * the calls to btrfs_invalidatepage() result in a huge amount of calls to
5545  * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
5546  * extent_state structures over and over, wasting lots of time.
5547  *
5548  * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
5549  * those expensive operations on a per page basis and do only the ordered io
5550  * finishing, while we release here the extent_map and extent_state structures,
5551  * without the excessive merging and splitting.
5552  */
5553 static void evict_inode_truncate_pages(struct inode *inode)
5554 {
5555         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
5556         struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree;
5557         struct rb_node *node;
5558
5559         ASSERT(inode->i_state & I_FREEING);
5560         truncate_inode_pages_final(&inode->i_data);
5561
5562         write_lock(&map_tree->lock);
5563         while (!RB_EMPTY_ROOT(&map_tree->map.rb_root)) {
5564                 struct extent_map *em;
5565
5566                 node = rb_first_cached(&map_tree->map);
5567                 em = rb_entry(node, struct extent_map, rb_node);
5568                 clear_bit(EXTENT_FLAG_PINNED, &em->flags);
5569                 clear_bit(EXTENT_FLAG_LOGGING, &em->flags);
5570                 remove_extent_mapping(map_tree, em);
5571                 free_extent_map(em);
5572                 if (need_resched()) {
5573                         write_unlock(&map_tree->lock);
5574                         cond_resched();
5575                         write_lock(&map_tree->lock);
5576                 }
5577         }
5578         write_unlock(&map_tree->lock);
5579
5580         /*
5581          * Keep looping until we have no more ranges in the io tree.
5582          * We can have ongoing bios started by readpages (called from readahead)
5583          * that have their endio callback (extent_io.c:end_bio_extent_readpage)
5584          * still in progress (unlocked the pages in the bio but did not yet
5585          * unlocked the ranges in the io tree). Therefore this means some
5586          * ranges can still be locked and eviction started because before
5587          * submitting those bios, which are executed by a separate task (work
5588          * queue kthread), inode references (inode->i_count) were not taken
5589          * (which would be dropped in the end io callback of each bio).
5590          * Therefore here we effectively end up waiting for those bios and
5591          * anyone else holding locked ranges without having bumped the inode's
5592          * reference count - if we don't do it, when they access the inode's
5593          * io_tree to unlock a range it may be too late, leading to an
5594          * use-after-free issue.
5595          */
5596         spin_lock(&io_tree->lock);
5597         while (!RB_EMPTY_ROOT(&io_tree->state)) {
5598                 struct extent_state *state;
5599                 struct extent_state *cached_state = NULL;
5600                 u64 start;
5601                 u64 end;
5602                 unsigned state_flags;
5603
5604                 node = rb_first(&io_tree->state);
5605                 state = rb_entry(node, struct extent_state, rb_node);
5606                 start = state->start;
5607                 end = state->end;
5608                 state_flags = state->state;
5609                 spin_unlock(&io_tree->lock);
5610
5611                 lock_extent_bits(io_tree, start, end, &cached_state);
5612
5613                 /*
5614                  * If still has DELALLOC flag, the extent didn't reach disk,
5615                  * and its reserved space won't be freed by delayed_ref.
5616                  * So we need to free its reserved space here.
5617                  * (Refer to comment in btrfs_invalidatepage, case 2)
5618                  *
5619                  * Note, end is the bytenr of last byte, so we need + 1 here.
5620                  */
5621                 if (state_flags & EXTENT_DELALLOC)
5622                         btrfs_qgroup_free_data(inode, NULL, start, end - start + 1);
5623
5624                 clear_extent_bit(io_tree, start, end,
5625                                  EXTENT_LOCKED | EXTENT_DELALLOC |
5626                                  EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1,
5627                                  &cached_state);
5628
5629                 cond_resched();
5630                 spin_lock(&io_tree->lock);
5631         }
5632         spin_unlock(&io_tree->lock);
5633 }
5634
5635 static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root,
5636                                                         struct btrfs_block_rsv *rsv)
5637 {
5638         struct btrfs_fs_info *fs_info = root->fs_info;
5639         struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5640         struct btrfs_trans_handle *trans;
5641         u64 delayed_refs_extra = btrfs_calc_insert_metadata_size(fs_info, 1);
5642         int ret;
5643
5644         /*
5645          * Eviction should be taking place at some place safe because of our
5646          * delayed iputs.  However the normal flushing code will run delayed
5647          * iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock.
5648          *
5649          * We reserve the delayed_refs_extra here again because we can't use
5650          * btrfs_start_transaction(root, 0) for the same deadlocky reason as
5651          * above.  We reserve our extra bit here because we generate a ton of
5652          * delayed refs activity by truncating.
5653          *
5654          * If we cannot make our reservation we'll attempt to steal from the
5655          * global reserve, because we really want to be able to free up space.
5656          */
5657         ret = btrfs_block_rsv_refill(root, rsv, rsv->size + delayed_refs_extra,
5658                                      BTRFS_RESERVE_FLUSH_EVICT);
5659         if (ret) {
5660                 /*
5661                  * Try to steal from the global reserve if there is space for
5662                  * it.
5663                  */
5664                 if (btrfs_check_space_for_delayed_refs(fs_info) ||
5665                     btrfs_block_rsv_migrate(global_rsv, rsv, rsv->size, 0)) {
5666                         btrfs_warn(fs_info,
5667                                    "could not allocate space for delete; will truncate on mount");
5668                         return ERR_PTR(-ENOSPC);
5669                 }
5670                 delayed_refs_extra = 0;
5671         }
5672
5673         trans = btrfs_join_transaction(root);
5674         if (IS_ERR(trans))
5675                 return trans;
5676
5677         if (delayed_refs_extra) {
5678                 trans->block_rsv = &fs_info->trans_block_rsv;
5679                 trans->bytes_reserved = delayed_refs_extra;
5680                 btrfs_block_rsv_migrate(rsv, trans->block_rsv,
5681                                         delayed_refs_extra, 1);
5682         }
5683         return trans;
5684 }
5685
5686 void btrfs_evict_inode(struct inode *inode)
5687 {
5688         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5689         struct btrfs_trans_handle *trans;
5690         struct btrfs_root *root = BTRFS_I(inode)->root;
5691         struct btrfs_block_rsv *rsv;
5692         int ret;
5693
5694         trace_btrfs_inode_evict(inode);
5695
5696         if (!root) {
5697                 clear_inode(inode);
5698                 return;
5699         }
5700
5701         evict_inode_truncate_pages(inode);
5702
5703         if (inode->i_nlink &&
5704             ((btrfs_root_refs(&root->root_item) != 0 &&
5705               root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
5706              btrfs_is_free_space_inode(BTRFS_I(inode))))
5707                 goto no_delete;
5708
5709         if (is_bad_inode(inode))
5710                 goto no_delete;
5711
5712         btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1);
5713
5714         if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags))
5715                 goto no_delete;
5716
5717         if (inode->i_nlink > 0) {
5718                 BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
5719                        root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID);
5720                 goto no_delete;
5721         }
5722
5723         ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
5724         if (ret)
5725                 goto no_delete;
5726
5727         rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
5728         if (!rsv)
5729                 goto no_delete;
5730         rsv->size = btrfs_calc_metadata_size(fs_info, 1);
5731         rsv->failfast = 1;
5732
5733         btrfs_i_size_write(BTRFS_I(inode), 0);
5734
5735         while (1) {
5736                 trans = evict_refill_and_join(root, rsv);
5737                 if (IS_ERR(trans))
5738                         goto free_rsv;
5739
5740                 trans->block_rsv = rsv;
5741
5742                 ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0);
5743                 trans->block_rsv = &fs_info->trans_block_rsv;
5744                 btrfs_end_transaction(trans);
5745                 btrfs_btree_balance_dirty(fs_info);
5746                 if (ret && ret != -ENOSPC && ret != -EAGAIN)
5747                         goto free_rsv;
5748                 else if (!ret)
5749                         break;
5750         }
5751
5752         /*
5753          * Errors here aren't a big deal, it just means we leave orphan items in
5754          * the tree. They will be cleaned up on the next mount. If the inode
5755          * number gets reused, cleanup deletes the orphan item without doing
5756          * anything, and unlink reuses the existing orphan item.
5757          *
5758          * If it turns out that we are dropping too many of these, we might want
5759          * to add a mechanism for retrying these after a commit.
5760          */
5761         trans = evict_refill_and_join(root, rsv);
5762         if (!IS_ERR(trans)) {
5763                 trans->block_rsv = rsv;
5764                 btrfs_orphan_del(trans, BTRFS_I(inode));
5765                 trans->block_rsv = &fs_info->trans_block_rsv;
5766                 btrfs_end_transaction(trans);
5767         }
5768
5769         if (!(root == fs_info->tree_root ||
5770               root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID))
5771                 btrfs_return_ino(root, btrfs_ino(BTRFS_I(inode)));
5772
5773 free_rsv:
5774         btrfs_free_block_rsv(fs_info, rsv);
5775 no_delete:
5776         /*
5777          * If we didn't successfully delete, the orphan item will still be in
5778          * the tree and we'll retry on the next mount. Again, we might also want
5779          * to retry these periodically in the future.
5780          */
5781         btrfs_remove_delayed_node(BTRFS_I(inode));
5782         clear_inode(inode);
5783 }
5784
5785 /*
5786  * Return the key found in the dir entry in the location pointer, fill @type
5787  * with BTRFS_FT_*, and return 0.
5788  *
5789  * If no dir entries were found, returns -ENOENT.
5790  * If found a corrupted location in dir entry, returns -EUCLEAN.
5791  */
5792 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry,
5793                                struct btrfs_key *location, u8 *type)
5794 {
5795         const char *name = dentry->d_name.name;
5796         int namelen = dentry->d_name.len;
5797         struct btrfs_dir_item *di;
5798         struct btrfs_path *path;
5799         struct btrfs_root *root = BTRFS_I(dir)->root;
5800         int ret = 0;
5801
5802         path = btrfs_alloc_path();
5803         if (!path)
5804                 return -ENOMEM;
5805
5806         di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)),
5807                         name, namelen, 0);
5808         if (IS_ERR_OR_NULL(di)) {
5809                 ret = di ? PTR_ERR(di) : -ENOENT;
5810                 goto out;
5811         }
5812
5813         btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
5814         if (location->type != BTRFS_INODE_ITEM_KEY &&
5815             location->type != BTRFS_ROOT_ITEM_KEY) {
5816                 ret = -EUCLEAN;
5817                 btrfs_warn(root->fs_info,
5818 "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))",
5819                            __func__, name, btrfs_ino(BTRFS_I(dir)),
5820                            location->objectid, location->type, location->offset);
5821         }
5822         if (!ret)
5823                 *type = btrfs_dir_type(path->nodes[0], di);
5824 out:
5825         btrfs_free_path(path);
5826         return ret;
5827 }
5828
5829 /*
5830  * when we hit a tree root in a directory, the btrfs part of the inode
5831  * needs to be changed to reflect the root directory of the tree root.  This
5832  * is kind of like crossing a mount point.
5833  */
5834 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,
5835                                     struct inode *dir,
5836                                     struct dentry *dentry,
5837                                     struct btrfs_key *location,
5838                                     struct btrfs_root **sub_root)
5839 {
5840         struct btrfs_path *path;
5841         struct btrfs_root *new_root;
5842         struct btrfs_root_ref *ref;
5843         struct extent_buffer *leaf;
5844         struct btrfs_key key;
5845         int ret;
5846         int err = 0;
5847
5848         path = btrfs_alloc_path();
5849         if (!path) {
5850                 err = -ENOMEM;
5851                 goto out;
5852         }
5853
5854         err = -ENOENT;
5855         key.objectid = BTRFS_I(dir)->root->root_key.objectid;
5856         key.type = BTRFS_ROOT_REF_KEY;
5857         key.offset = location->objectid;
5858
5859         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
5860         if (ret) {
5861                 if (ret < 0)
5862                         err = ret;
5863                 goto out;
5864         }
5865
5866         leaf = path->nodes[0];
5867         ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
5868         if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) ||
5869             btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len)
5870                 goto out;
5871
5872         ret = memcmp_extent_buffer(leaf, dentry->d_name.name,
5873                                    (unsigned long)(ref + 1),
5874                                    dentry->d_name.len);
5875         if (ret)
5876                 goto out;
5877
5878         btrfs_release_path(path);
5879
5880         new_root = btrfs_read_fs_root_no_name(fs_info, location);
5881         if (IS_ERR(new_root)) {
5882                 err = PTR_ERR(new_root);
5883                 goto out;
5884         }
5885
5886         *sub_root = new_root;
5887         location->objectid = btrfs_root_dirid(&new_root->root_item);
5888         location->type = BTRFS_INODE_ITEM_KEY;
5889         location->offset = 0;
5890         err = 0;
5891 out:
5892         btrfs_free_path(path);
5893         return err;
5894 }
5895
5896 static void inode_tree_add(struct inode *inode)
5897 {
5898         struct btrfs_root *root = BTRFS_I(inode)->root;
5899         struct btrfs_inode *entry;
5900         struct rb_node **p;
5901         struct rb_node *parent;
5902         struct rb_node *new = &BTRFS_I(inode)->rb_node;
5903         u64 ino = btrfs_ino(BTRFS_I(inode));
5904
5905         if (inode_unhashed(inode))
5906                 return;
5907         parent = NULL;
5908         spin_lock(&root->inode_lock);
5909         p = &root->inode_tree.rb_node;
5910         while (*p) {
5911                 parent = *p;
5912                 entry = rb_entry(parent, struct btrfs_inode, rb_node);
5913
5914                 if (ino < btrfs_ino(entry))
5915                         p = &parent->rb_left;
5916                 else if (ino > btrfs_ino(entry))
5917                         p = &parent->rb_right;
5918                 else {
5919                         WARN_ON(!(entry->vfs_inode.i_state &
5920                                   (I_WILL_FREE | I_FREEING)));
5921                         rb_replace_node(parent, new, &root->inode_tree);
5922                         RB_CLEAR_NODE(parent);
5923                         spin_unlock(&root->inode_lock);
5924                         return;
5925                 }
5926         }
5927         rb_link_node(new, parent, p);
5928         rb_insert_color(new, &root->inode_tree);
5929         spin_unlock(&root->inode_lock);
5930 }
5931
5932 static void inode_tree_del(struct inode *inode)
5933 {
5934         struct btrfs_root *root = BTRFS_I(inode)->root;
5935         int empty = 0;
5936
5937         spin_lock(&root->inode_lock);
5938         if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) {
5939                 rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree);
5940                 RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
5941                 empty = RB_EMPTY_ROOT(&root->inode_tree);
5942         }
5943         spin_unlock(&root->inode_lock);
5944
5945         if (empty && btrfs_root_refs(&root->root_item) == 0) {
5946                 spin_lock(&root->inode_lock);
5947                 empty = RB_EMPTY_ROOT(&root->inode_tree);
5948                 spin_unlock(&root->inode_lock);
5949                 if (empty)
5950                         btrfs_add_dead_root(root);
5951         }
5952 }
5953
5954
5955 static int btrfs_init_locked_inode(struct inode *inode, void *p)
5956 {
5957         struct btrfs_iget_args *args = p;
5958         inode->i_ino = args->location->objectid;
5959         memcpy(&BTRFS_I(inode)->location, args->location,
5960                sizeof(*args->location));
5961         BTRFS_I(inode)->root = args->root;
5962         return 0;
5963 }
5964
5965 static int btrfs_find_actor(struct inode *inode, void *opaque)
5966 {
5967         struct btrfs_iget_args *args = opaque;
5968         return args->location->objectid == BTRFS_I(inode)->location.objectid &&
5969                 args->root == BTRFS_I(inode)->root;
5970 }
5971
5972 static struct inode *btrfs_iget_locked(struct super_block *s,
5973                                        struct btrfs_key *location,
5974                                        struct btrfs_root *root)
5975 {
5976         struct inode *inode;
5977         struct btrfs_iget_args args;
5978         unsigned long hashval = btrfs_inode_hash(location->objectid, root);
5979
5980         args.location = location;
5981         args.root = root;
5982
5983         inode = iget5_locked(s, hashval, btrfs_find_actor,
5984                              btrfs_init_locked_inode,
5985                              (void *)&args);
5986         return inode;
5987 }
5988
5989 /* Get an inode object given its location and corresponding root.
5990  * Returns in *is_new if the inode was read from disk
5991  */
5992 struct inode *btrfs_iget_path(struct super_block *s, struct btrfs_key *location,
5993                               struct btrfs_root *root, int *new,
5994                               struct btrfs_path *path)
5995 {
5996         struct inode *inode;
5997
5998         inode = btrfs_iget_locked(s, location, root);
5999         if (!inode)
6000                 return ERR_PTR(-ENOMEM);
6001
6002         if (inode->i_state & I_NEW) {
6003                 int ret;
6004
6005                 ret = btrfs_read_locked_inode(inode, path);
6006                 if (!ret) {
6007                         inode_tree_add(inode);
6008                         unlock_new_inode(inode);
6009                         if (new)
6010                                 *new = 1;
6011                 } else {
6012                         iget_failed(inode);
6013                         /*
6014                          * ret > 0 can come from btrfs_search_slot called by
6015                          * btrfs_read_locked_inode, this means the inode item
6016                          * was not found.
6017                          */
6018                         if (ret > 0)
6019                                 ret = -ENOENT;
6020                         inode = ERR_PTR(ret);
6021                 }
6022         }
6023
6024         return inode;
6025 }
6026
6027 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
6028                          struct btrfs_root *root, int *new)
6029 {
6030         return btrfs_iget_path(s, location, root, new, NULL);
6031 }
6032
6033 static struct inode *new_simple_dir(struct super_block *s,
6034                                     struct btrfs_key *key,
6035                                     struct btrfs_root *root)
6036 {
6037         struct inode *inode = new_inode(s);
6038
6039         if (!inode)
6040                 return ERR_PTR(-ENOMEM);
6041
6042         BTRFS_I(inode)->root = root;
6043         memcpy(&BTRFS_I(inode)->location, key, sizeof(*key));
6044         set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
6045
6046         inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID;
6047         inode->i_op = &btrfs_dir_ro_inode_operations;
6048         inode->i_opflags &= ~IOP_XATTR;
6049         inode->i_fop = &simple_dir_operations;
6050         inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
6051         inode->i_mtime = current_time(inode);
6052         inode->i_atime = inode->i_mtime;
6053         inode->i_ctime = inode->i_mtime;
6054         BTRFS_I(inode)->i_otime = inode->i_mtime;
6055
6056         return inode;
6057 }
6058
6059 static inline u8 btrfs_inode_type(struct inode *inode)
6060 {
6061         /*
6062          * Compile-time asserts that generic FT_* types still match
6063          * BTRFS_FT_* types
6064          */
6065         BUILD_BUG_ON(BTRFS_FT_UNKNOWN != FT_UNKNOWN);
6066         BUILD_BUG_ON(BTRFS_FT_REG_FILE != FT_REG_FILE);
6067         BUILD_BUG_ON(BTRFS_FT_DIR != FT_DIR);
6068         BUILD_BUG_ON(BTRFS_FT_CHRDEV != FT_CHRDEV);
6069         BUILD_BUG_ON(BTRFS_FT_BLKDEV != FT_BLKDEV);
6070         BUILD_BUG_ON(BTRFS_FT_FIFO != FT_FIFO);
6071         BUILD_BUG_ON(BTRFS_FT_SOCK != FT_SOCK);
6072         BUILD_BUG_ON(BTRFS_FT_SYMLINK != FT_SYMLINK);
6073
6074         return fs_umode_to_ftype(inode->i_mode);
6075 }
6076
6077 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
6078 {
6079         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6080         struct inode *inode;
6081         struct btrfs_root *root = BTRFS_I(dir)->root;
6082         struct btrfs_root *sub_root = root;
6083         struct btrfs_key location;
6084         u8 di_type = 0;
6085         int index;
6086         int ret = 0;
6087
6088         if (dentry->d_name.len > BTRFS_NAME_LEN)
6089                 return ERR_PTR(-ENAMETOOLONG);
6090
6091         ret = btrfs_inode_by_name(dir, dentry, &location, &di_type);
6092         if (ret < 0)
6093                 return ERR_PTR(ret);
6094
6095         if (location.type == BTRFS_INODE_ITEM_KEY) {
6096                 inode = btrfs_iget(dir->i_sb, &location, root, NULL);
6097                 if (IS_ERR(inode))
6098                         return inode;
6099
6100                 /* Do extra check against inode mode with di_type */
6101                 if (btrfs_inode_type(inode) != di_type) {
6102                         btrfs_crit(fs_info,
6103 "inode mode mismatch with dir: inode mode=0%o btrfs type=%u dir type=%u",
6104                                   inode->i_mode, btrfs_inode_type(inode),
6105                                   di_type);
6106                         iput(inode);
6107                         return ERR_PTR(-EUCLEAN);
6108                 }
6109                 return inode;
6110         }
6111
6112         index = srcu_read_lock(&fs_info->subvol_srcu);
6113         ret = fixup_tree_root_location(fs_info, dir, dentry,
6114                                        &location, &sub_root);
6115         if (ret < 0) {
6116                 if (ret != -ENOENT)
6117                         inode = ERR_PTR(ret);
6118                 else
6119                         inode = new_simple_dir(dir->i_sb, &location, sub_root);
6120         } else {
6121                 inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL);
6122         }
6123         srcu_read_unlock(&fs_info->subvol_srcu, index);
6124
6125         if (!IS_ERR(inode) && root != sub_root) {
6126                 down_read(&fs_info->cleanup_work_sem);
6127                 if (!sb_rdonly(inode->i_sb))
6128                         ret = btrfs_orphan_cleanup(sub_root);
6129                 up_read(&fs_info->cleanup_work_sem);
6130                 if (ret) {
6131                         iput(inode);
6132                         inode = ERR_PTR(ret);
6133                 }
6134         }
6135
6136         return inode;
6137 }
6138
6139 static int btrfs_dentry_delete(const struct dentry *dentry)
6140 {
6141         struct btrfs_root *root;
6142         struct inode *inode = d_inode(dentry);
6143
6144         if (!inode && !IS_ROOT(dentry))
6145                 inode = d_inode(dentry->d_parent);
6146
6147         if (inode) {
6148                 root = BTRFS_I(inode)->root;
6149                 if (btrfs_root_refs(&root->root_item) == 0)
6150                         return 1;
6151
6152                 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
6153                         return 1;
6154         }
6155         return 0;
6156 }
6157
6158 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
6159                                    unsigned int flags)
6160 {
6161         struct inode *inode = btrfs_lookup_dentry(dir, dentry);
6162
6163         if (inode == ERR_PTR(-ENOENT))
6164                 inode = NULL;
6165         return d_splice_alias(inode, dentry);
6166 }
6167
6168 /*
6169  * All this infrastructure exists because dir_emit can fault, and we are holding
6170  * the tree lock when doing readdir.  For now just allocate a buffer and copy
6171  * our information into that, and then dir_emit from the buffer.  This is
6172  * similar to what NFS does, only we don't keep the buffer around in pagecache
6173  * because I'm afraid I'll mess that up.  Long term we need to make filldir do
6174  * copy_to_user_inatomic so we don't have to worry about page faulting under the
6175  * tree lock.
6176  */
6177 static int btrfs_opendir(struct inode *inode, struct file *file)
6178 {
6179         struct btrfs_file_private *private;
6180
6181         private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL);
6182         if (!private)
6183                 return -ENOMEM;
6184         private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
6185         if (!private->filldir_buf) {
6186                 kfree(private);
6187                 return -ENOMEM;
6188         }
6189         file->private_data = private;
6190         return 0;
6191 }
6192
6193 struct dir_entry {
6194         u64 ino;
6195         u64 offset;
6196         unsigned type;
6197         int name_len;
6198 };
6199
6200 static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx)
6201 {
6202         while (entries--) {
6203                 struct dir_entry *entry = addr;
6204                 char *name = (char *)(entry + 1);
6205
6206                 ctx->pos = get_unaligned(&entry->offset);
6207                 if (!dir_emit(ctx, name, get_unaligned(&entry->name_len),
6208                                          get_unaligned(&entry->ino),
6209                                          get_unaligned(&entry->type)))
6210                         return 1;
6211                 addr += sizeof(struct dir_entry) +
6212                         get_unaligned(&entry->name_len);
6213                 ctx->pos++;
6214         }
6215         return 0;
6216 }
6217
6218 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
6219 {
6220         struct inode *inode = file_inode(file);
6221         struct btrfs_root *root = BTRFS_I(inode)->root;
6222         struct btrfs_file_private *private = file->private_data;
6223         struct btrfs_dir_item *di;
6224         struct btrfs_key key;
6225         struct btrfs_key found_key;
6226         struct btrfs_path *path;
6227         void *addr;
6228         struct list_head ins_list;
6229         struct list_head del_list;
6230         int ret;
6231         struct extent_buffer *leaf;
6232         int slot;
6233         char *name_ptr;
6234         int name_len;
6235         int entries = 0;
6236         int total_len = 0;
6237         bool put = false;
6238         struct btrfs_key location;
6239
6240         if (!dir_emit_dots(file, ctx))
6241                 return 0;
6242
6243         path = btrfs_alloc_path();
6244         if (!path)
6245                 return -ENOMEM;
6246
6247         addr = private->filldir_buf;
6248         path->reada = READA_FORWARD;
6249
6250         INIT_LIST_HEAD(&ins_list);
6251         INIT_LIST_HEAD(&del_list);
6252         put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list);
6253
6254 again:
6255         key.type = BTRFS_DIR_INDEX_KEY;
6256         key.offset = ctx->pos;
6257         key.objectid = btrfs_ino(BTRFS_I(inode));
6258
6259         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6260         if (ret < 0)
6261                 goto err;
6262
6263         while (1) {
6264                 struct dir_entry *entry;
6265
6266                 leaf = path->nodes[0];
6267                 slot = path->slots[0];
6268                 if (slot >= btrfs_header_nritems(leaf)) {
6269                         ret = btrfs_next_leaf(root, path);
6270                         if (ret < 0)
6271                                 goto err;
6272                         else if (ret > 0)
6273                                 break;
6274                         continue;
6275                 }
6276
6277                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
6278
6279                 if (found_key.objectid != key.objectid)
6280                         break;
6281                 if (found_key.type != BTRFS_DIR_INDEX_KEY)
6282                         break;
6283                 if (found_key.offset < ctx->pos)
6284                         goto next;
6285                 if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
6286                         goto next;
6287                 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
6288                 name_len = btrfs_dir_name_len(leaf, di);
6289                 if ((total_len + sizeof(struct dir_entry) + name_len) >=
6290                     PAGE_SIZE) {
6291                         btrfs_release_path(path);
6292                         ret = btrfs_filldir(private->filldir_buf, entries, ctx);
6293                         if (ret)
6294                                 goto nopos;
6295                         addr = private->filldir_buf;
6296                         entries = 0;
6297                         total_len = 0;
6298                         goto again;
6299                 }
6300
6301                 entry = addr;
6302                 put_unaligned(name_len, &entry->name_len);
6303                 name_ptr = (char *)(entry + 1);
6304                 read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1),
6305                                    name_len);
6306                 put_unaligned(fs_ftype_to_dtype(btrfs_dir_type(leaf, di)),
6307                                 &entry->type);
6308                 btrfs_dir_item_key_to_cpu(leaf, di, &location);
6309                 put_unaligned(location.objectid, &entry->ino);
6310                 put_unaligned(found_key.offset, &entry->offset);
6311                 entries++;
6312                 addr += sizeof(struct dir_entry) + name_len;
6313                 total_len += sizeof(struct dir_entry) + name_len;
6314 next:
6315                 path->slots[0]++;
6316         }
6317         btrfs_release_path(path);
6318
6319         ret = btrfs_filldir(private->filldir_buf, entries, ctx);
6320         if (ret)
6321                 goto nopos;
6322
6323         ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list);
6324         if (ret)
6325                 goto nopos;
6326
6327         /*
6328          * Stop new entries from being returned after we return the last
6329          * entry.
6330          *
6331          * New directory entries are assigned a strictly increasing
6332          * offset.  This means that new entries created during readdir
6333          * are *guaranteed* to be seen in the future by that readdir.
6334          * This has broken buggy programs which operate on names as
6335          * they're returned by readdir.  Until we re-use freed offsets
6336          * we have this hack to stop new entries from being returned
6337          * under the assumption that they'll never reach this huge
6338          * offset.
6339          *
6340          * This is being careful not to overflow 32bit loff_t unless the
6341          * last entry requires it because doing so has broken 32bit apps
6342          * in the past.
6343          */
6344         if (ctx->pos >= INT_MAX)
6345                 ctx->pos = LLONG_MAX;
6346         else
6347                 ctx->pos = INT_MAX;
6348 nopos:
6349         ret = 0;
6350 err:
6351         if (put)
6352                 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list);
6353         btrfs_free_path(path);
6354         return ret;
6355 }
6356
6357 /*
6358  * This is somewhat expensive, updating the tree every time the
6359  * inode changes.  But, it is most likely to find the inode in cache.
6360  * FIXME, needs more benchmarking...there are no reasons other than performance
6361  * to keep or drop this code.
6362  */
6363 static int btrfs_dirty_inode(struct inode *inode)
6364 {
6365         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6366         struct btrfs_root *root = BTRFS_I(inode)->root;
6367         struct btrfs_trans_handle *trans;
6368         int ret;
6369
6370         if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
6371                 return 0;
6372
6373         trans = btrfs_join_transaction(root);
6374         if (IS_ERR(trans))
6375                 return PTR_ERR(trans);
6376
6377         ret = btrfs_update_inode(trans, root, inode);
6378         if (ret && (ret == -ENOSPC || ret == -EDQUOT)) {
6379                 /* whoops, lets try again with the full transaction */
6380                 btrfs_end_transaction(trans);
6381                 trans = btrfs_start_transaction(root, 1);
6382                 if (IS_ERR(trans))
6383                         return PTR_ERR(trans);
6384
6385                 ret = btrfs_update_inode(trans, root, inode);
6386         }
6387         btrfs_end_transaction(trans);
6388         if (BTRFS_I(inode)->delayed_node)
6389                 btrfs_balance_delayed_items(fs_info);
6390
6391         return ret;
6392 }
6393
6394 /*
6395  * This is a copy of file_update_time.  We need this so we can return error on
6396  * ENOSPC for updating the inode in the case of file write and mmap writes.
6397  */
6398 static int btrfs_update_time(struct inode *inode, struct timespec64 *now,
6399                              int flags)
6400 {
6401         struct btrfs_root *root = BTRFS_I(inode)->root;
6402         bool dirty = flags & ~S_VERSION;
6403
6404         if (btrfs_root_readonly(root))
6405                 return -EROFS;
6406
6407         if (flags & S_VERSION)
6408                 dirty |= inode_maybe_inc_iversion(inode, dirty);
6409         if (flags & S_CTIME)
6410                 inode->i_ctime = *now;
6411         if (flags & S_MTIME)
6412                 inode->i_mtime = *now;
6413         if (flags & S_ATIME)
6414                 inode->i_atime = *now;
6415         return dirty ? btrfs_dirty_inode(inode) : 0;
6416 }
6417
6418 /*
6419  * find the highest existing sequence number in a directory
6420  * and then set the in-memory index_cnt variable to reflect
6421  * free sequence numbers
6422  */
6423 static int btrfs_set_inode_index_count(struct btrfs_inode *inode)
6424 {
6425         struct btrfs_root *root = inode->root;
6426         struct btrfs_key key, found_key;
6427         struct btrfs_path *path;
6428         struct extent_buffer *leaf;
6429         int ret;
6430
6431         key.objectid = btrfs_ino(inode);
6432         key.type = BTRFS_DIR_INDEX_KEY;
6433         key.offset = (u64)-1;
6434
6435         path = btrfs_alloc_path();
6436         if (!path)
6437                 return -ENOMEM;
6438
6439         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6440         if (ret < 0)
6441                 goto out;
6442         /* FIXME: we should be able to handle this */
6443         if (ret == 0)
6444                 goto out;
6445         ret = 0;
6446
6447         /*
6448          * MAGIC NUMBER EXPLANATION:
6449          * since we search a directory based on f_pos we have to start at 2
6450          * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
6451          * else has to start at 2
6452          */
6453         if (path->slots[0] == 0) {
6454                 inode->index_cnt = 2;
6455                 goto out;
6456         }
6457
6458         path->slots[0]--;
6459
6460         leaf = path->nodes[0];
6461         btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6462
6463         if (found_key.objectid != btrfs_ino(inode) ||
6464             found_key.type != BTRFS_DIR_INDEX_KEY) {
6465                 inode->index_cnt = 2;
6466                 goto out;
6467         }
6468
6469         inode->index_cnt = found_key.offset + 1;
6470 out:
6471         btrfs_free_path(path);
6472         return ret;
6473 }
6474
6475 /*
6476  * helper to find a free sequence number in a given directory.  This current
6477  * code is very simple, later versions will do smarter things in the btree
6478  */
6479 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index)
6480 {
6481         int ret = 0;
6482
6483         if (dir->index_cnt == (u64)-1) {
6484                 ret = btrfs_inode_delayed_dir_index_count(dir);
6485                 if (ret) {
6486                         ret = btrfs_set_inode_index_count(dir);
6487                         if (ret)
6488                                 return ret;
6489                 }
6490         }
6491
6492         *index = dir->index_cnt;
6493         dir->index_cnt++;
6494
6495         return ret;
6496 }
6497
6498 static int btrfs_insert_inode_locked(struct inode *inode)
6499 {
6500         struct btrfs_iget_args args;
6501         args.location = &BTRFS_I(inode)->location;
6502         args.root = BTRFS_I(inode)->root;
6503
6504         return insert_inode_locked4(inode,
6505                    btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
6506                    btrfs_find_actor, &args);
6507 }
6508
6509 /*
6510  * Inherit flags from the parent inode.
6511  *
6512  * Currently only the compression flags and the cow flags are inherited.
6513  */
6514 static void btrfs_inherit_iflags(struct inode *inode, struct inode *dir)
6515 {
6516         unsigned int flags;
6517
6518         if (!dir)
6519                 return;
6520
6521         flags = BTRFS_I(dir)->flags;
6522
6523         if (flags & BTRFS_INODE_NOCOMPRESS) {
6524                 BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS;
6525                 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
6526         } else if (flags & BTRFS_INODE_COMPRESS) {
6527                 BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS;
6528                 BTRFS_I(inode)->flags |= BTRFS_INODE_COMPRESS;
6529         }
6530
6531         if (flags & BTRFS_INODE_NODATACOW) {
6532                 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW;
6533                 if (S_ISREG(inode->i_mode))
6534                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6535         }
6536
6537         btrfs_sync_inode_flags_to_i_flags(inode);
6538 }
6539
6540 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans,
6541                                      struct btrfs_root *root,
6542                                      struct inode *dir,
6543                                      const char *name, int name_len,
6544                                      u64 ref_objectid, u64 objectid,
6545                                      umode_t mode, u64 *index)
6546 {
6547         struct btrfs_fs_info *fs_info = root->fs_info;
6548         struct inode *inode;
6549         struct btrfs_inode_item *inode_item;
6550         struct btrfs_key *location;
6551         struct btrfs_path *path;
6552         struct btrfs_inode_ref *ref;
6553         struct btrfs_key key[2];
6554         u32 sizes[2];
6555         int nitems = name ? 2 : 1;
6556         unsigned long ptr;
6557         unsigned int nofs_flag;
6558         int ret;
6559
6560         path = btrfs_alloc_path();
6561         if (!path)
6562                 return ERR_PTR(-ENOMEM);
6563
6564         nofs_flag = memalloc_nofs_save();
6565         inode = new_inode(fs_info->sb);
6566         memalloc_nofs_restore(nofs_flag);
6567         if (!inode) {
6568                 btrfs_free_path(path);
6569                 return ERR_PTR(-ENOMEM);
6570         }
6571
6572         /*
6573          * O_TMPFILE, set link count to 0, so that after this point,
6574          * we fill in an inode item with the correct link count.
6575          */
6576         if (!name)
6577                 set_nlink(inode, 0);
6578
6579         /*
6580          * we have to initialize this early, so we can reclaim the inode
6581          * number if we fail afterwards in this function.
6582          */
6583         inode->i_ino = objectid;
6584
6585         if (dir && name) {
6586                 trace_btrfs_inode_request(dir);
6587
6588                 ret = btrfs_set_inode_index(BTRFS_I(dir), index);
6589                 if (ret) {
6590                         btrfs_free_path(path);
6591                         iput(inode);
6592                         return ERR_PTR(ret);
6593                 }
6594         } else if (dir) {
6595                 *index = 0;
6596         }
6597         /*
6598          * index_cnt is ignored for everything but a dir,
6599          * btrfs_set_inode_index_count has an explanation for the magic
6600          * number
6601          */
6602         BTRFS_I(inode)->index_cnt = 2;
6603         BTRFS_I(inode)->dir_index = *index;
6604         BTRFS_I(inode)->root = root;
6605         BTRFS_I(inode)->generation = trans->transid;
6606         inode->i_generation = BTRFS_I(inode)->generation;
6607
6608         /*
6609          * We could have gotten an inode number from somebody who was fsynced
6610          * and then removed in this same transaction, so let's just set full
6611          * sync since it will be a full sync anyway and this will blow away the
6612          * old info in the log.
6613          */
6614         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
6615
6616         key[0].objectid = objectid;
6617         key[0].type = BTRFS_INODE_ITEM_KEY;
6618         key[0].offset = 0;
6619
6620         sizes[0] = sizeof(struct btrfs_inode_item);
6621
6622         if (name) {
6623                 /*
6624                  * Start new inodes with an inode_ref. This is slightly more
6625                  * efficient for small numbers of hard links since they will
6626                  * be packed into one item. Extended refs will kick in if we
6627                  * add more hard links than can fit in the ref item.
6628                  */
6629                 key[1].objectid = objectid;
6630                 key[1].type = BTRFS_INODE_REF_KEY;
6631                 key[1].offset = ref_objectid;
6632
6633                 sizes[1] = name_len + sizeof(*ref);
6634         }
6635
6636         location = &BTRFS_I(inode)->location;
6637         location->objectid = objectid;
6638         location->offset = 0;
6639         location->type = BTRFS_INODE_ITEM_KEY;
6640
6641         ret = btrfs_insert_inode_locked(inode);
6642         if (ret < 0) {
6643                 iput(inode);
6644                 goto fail;
6645         }
6646
6647         path->leave_spinning = 1;
6648         ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems);
6649         if (ret != 0)
6650                 goto fail_unlock;
6651
6652         inode_init_owner(inode, dir, mode);
6653         inode_set_bytes(inode, 0);
6654
6655         inode->i_mtime = current_time(inode);
6656         inode->i_atime = inode->i_mtime;
6657         inode->i_ctime = inode->i_mtime;
6658         BTRFS_I(inode)->i_otime = inode->i_mtime;
6659
6660         inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
6661                                   struct btrfs_inode_item);
6662         memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item,
6663                              sizeof(*inode_item));
6664         fill_inode_item(trans, path->nodes[0], inode_item, inode);
6665
6666         if (name) {
6667                 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
6668                                      struct btrfs_inode_ref);
6669                 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len);
6670                 btrfs_set_inode_ref_index(path->nodes[0], ref, *index);
6671                 ptr = (unsigned long)(ref + 1);
6672                 write_extent_buffer(path->nodes[0], name, ptr, name_len);
6673         }
6674
6675         btrfs_mark_buffer_dirty(path->nodes[0]);
6676         btrfs_free_path(path);
6677
6678         btrfs_inherit_iflags(inode, dir);
6679
6680         if (S_ISREG(mode)) {
6681                 if (btrfs_test_opt(fs_info, NODATASUM))
6682                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6683                 if (btrfs_test_opt(fs_info, NODATACOW))
6684                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
6685                                 BTRFS_INODE_NODATASUM;
6686         }
6687
6688         inode_tree_add(inode);
6689
6690         trace_btrfs_inode_new(inode);
6691         btrfs_set_inode_last_trans(trans, inode);
6692
6693         btrfs_update_root_times(trans, root);
6694
6695         ret = btrfs_inode_inherit_props(trans, inode, dir);
6696         if (ret)
6697                 btrfs_err(fs_info,
6698                           "error inheriting props for ino %llu (root %llu): %d",
6699                         btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret);
6700
6701         return inode;
6702
6703 fail_unlock:
6704         discard_new_inode(inode);
6705 fail:
6706         if (dir && name)
6707                 BTRFS_I(dir)->index_cnt--;
6708         btrfs_free_path(path);
6709         return ERR_PTR(ret);
6710 }
6711
6712 /*
6713  * utility function to add 'inode' into 'parent_inode' with
6714  * a give name and a given sequence number.
6715  * if 'add_backref' is true, also insert a backref from the
6716  * inode to the parent directory.
6717  */
6718 int btrfs_add_link(struct btrfs_trans_handle *trans,
6719                    struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
6720                    const char *name, int name_len, int add_backref, u64 index)
6721 {
6722         int ret = 0;
6723         struct btrfs_key key;
6724         struct btrfs_root *root = parent_inode->root;
6725         u64 ino = btrfs_ino(inode);
6726         u64 parent_ino = btrfs_ino(parent_inode);
6727
6728         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6729                 memcpy(&key, &inode->root->root_key, sizeof(key));
6730         } else {
6731                 key.objectid = ino;
6732                 key.type = BTRFS_INODE_ITEM_KEY;
6733                 key.offset = 0;
6734         }
6735
6736         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6737                 ret = btrfs_add_root_ref(trans, key.objectid,
6738                                          root->root_key.objectid, parent_ino,
6739                                          index, name, name_len);
6740         } else if (add_backref) {
6741                 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino,
6742                                              parent_ino, index);
6743         }
6744
6745         /* Nothing to clean up yet */
6746         if (ret)
6747                 return ret;
6748
6749         ret = btrfs_insert_dir_item(trans, name, name_len, parent_inode, &key,
6750                                     btrfs_inode_type(&inode->vfs_inode), index);
6751         if (ret == -EEXIST || ret == -EOVERFLOW)
6752                 goto fail_dir_item;
6753         else if (ret) {
6754                 btrfs_abort_transaction(trans, ret);
6755                 return ret;
6756         }
6757
6758         btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size +
6759                            name_len * 2);
6760         inode_inc_iversion(&parent_inode->vfs_inode);
6761         /*
6762          * If we are replaying a log tree, we do not want to update the mtime
6763          * and ctime of the parent directory with the current time, since the
6764          * log replay procedure is responsible for setting them to their correct
6765          * values (the ones it had when the fsync was done).
6766          */
6767         if (!test_bit(BTRFS_FS_LOG_RECOVERING, &root->fs_info->flags)) {
6768                 struct timespec64 now = current_time(&parent_inode->vfs_inode);
6769
6770                 parent_inode->vfs_inode.i_mtime = now;
6771                 parent_inode->vfs_inode.i_ctime = now;
6772         }
6773         ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode);
6774         if (ret)
6775                 btrfs_abort_transaction(trans, ret);
6776         return ret;
6777
6778 fail_dir_item:
6779         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6780                 u64 local_index;
6781                 int err;
6782                 err = btrfs_del_root_ref(trans, key.objectid,
6783                                          root->root_key.objectid, parent_ino,
6784                                          &local_index, name, name_len);
6785                 if (err)
6786                         btrfs_abort_transaction(trans, err);
6787         } else if (add_backref) {
6788                 u64 local_index;
6789                 int err;
6790
6791                 err = btrfs_del_inode_ref(trans, root, name, name_len,
6792                                           ino, parent_ino, &local_index);
6793                 if (err)
6794                         btrfs_abort_transaction(trans, err);
6795         }
6796
6797         /* Return the original error code */
6798         return ret;
6799 }
6800
6801 static int btrfs_add_nondir(struct btrfs_trans_handle *trans,
6802                             struct btrfs_inode *dir, struct dentry *dentry,
6803                             struct btrfs_inode *inode, int backref, u64 index)
6804 {
6805         int err = btrfs_add_link(trans, dir, inode,
6806                                  dentry->d_name.name, dentry->d_name.len,
6807                                  backref, index);
6808         if (err > 0)
6809                 err = -EEXIST;
6810         return err;
6811 }
6812
6813 static int btrfs_mknod(struct inode *dir, struct dentry *dentry,
6814                         umode_t mode, dev_t rdev)
6815 {
6816         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6817         struct btrfs_trans_handle *trans;
6818         struct btrfs_root *root = BTRFS_I(dir)->root;
6819         struct inode *inode = NULL;
6820         int err;
6821         u64 objectid;
6822         u64 index = 0;
6823
6824         /*
6825          * 2 for inode item and ref
6826          * 2 for dir items
6827          * 1 for xattr if selinux is on
6828          */
6829         trans = btrfs_start_transaction(root, 5);
6830         if (IS_ERR(trans))
6831                 return PTR_ERR(trans);
6832
6833         err = btrfs_find_free_objectid(root, &objectid);
6834         if (err)
6835                 goto out_unlock;
6836
6837         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6838                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6839                         mode, &index);
6840         if (IS_ERR(inode)) {
6841                 err = PTR_ERR(inode);
6842                 inode = NULL;
6843                 goto out_unlock;
6844         }
6845
6846         /*
6847         * If the active LSM wants to access the inode during
6848         * d_instantiate it needs these. Smack checks to see
6849         * if the filesystem supports xattrs by looking at the
6850         * ops vector.
6851         */
6852         inode->i_op = &btrfs_special_inode_operations;
6853         init_special_inode(inode, inode->i_mode, rdev);
6854
6855         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6856         if (err)
6857                 goto out_unlock;
6858
6859         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6860                         0, index);
6861         if (err)
6862                 goto out_unlock;
6863
6864         btrfs_update_inode(trans, root, inode);
6865         d_instantiate_new(dentry, inode);
6866
6867 out_unlock:
6868         btrfs_end_transaction(trans);
6869         btrfs_btree_balance_dirty(fs_info);
6870         if (err && inode) {
6871                 inode_dec_link_count(inode);
6872                 discard_new_inode(inode);
6873         }
6874         return err;
6875 }
6876
6877 static int btrfs_create(struct inode *dir, struct dentry *dentry,
6878                         umode_t mode, bool excl)
6879 {
6880         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6881         struct btrfs_trans_handle *trans;
6882         struct btrfs_root *root = BTRFS_I(dir)->root;
6883         struct inode *inode = NULL;
6884         int err;
6885         u64 objectid;
6886         u64 index = 0;
6887
6888         /*
6889          * 2 for inode item and ref
6890          * 2 for dir items
6891          * 1 for xattr if selinux is on
6892          */
6893         trans = btrfs_start_transaction(root, 5);
6894         if (IS_ERR(trans))
6895                 return PTR_ERR(trans);
6896
6897         err = btrfs_find_free_objectid(root, &objectid);
6898         if (err)
6899                 goto out_unlock;
6900
6901         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6902                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6903                         mode, &index);
6904         if (IS_ERR(inode)) {
6905                 err = PTR_ERR(inode);
6906                 inode = NULL;
6907                 goto out_unlock;
6908         }
6909         /*
6910         * If the active LSM wants to access the inode during
6911         * d_instantiate it needs these. Smack checks to see
6912         * if the filesystem supports xattrs by looking at the
6913         * ops vector.
6914         */
6915         inode->i_fop = &btrfs_file_operations;
6916         inode->i_op = &btrfs_file_inode_operations;
6917         inode->i_mapping->a_ops = &btrfs_aops;
6918
6919         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6920         if (err)
6921                 goto out_unlock;
6922
6923         err = btrfs_update_inode(trans, root, inode);
6924         if (err)
6925                 goto out_unlock;
6926
6927         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6928                         0, index);
6929         if (err)
6930                 goto out_unlock;
6931
6932         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
6933         d_instantiate_new(dentry, inode);
6934
6935 out_unlock:
6936         btrfs_end_transaction(trans);
6937         if (err && inode) {
6938                 inode_dec_link_count(inode);
6939                 discard_new_inode(inode);
6940         }
6941         btrfs_btree_balance_dirty(fs_info);
6942         return err;
6943 }
6944
6945 static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
6946                       struct dentry *dentry)
6947 {
6948         struct btrfs_trans_handle *trans = NULL;
6949         struct btrfs_root *root = BTRFS_I(dir)->root;
6950         struct inode *inode = d_inode(old_dentry);
6951         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6952         u64 index;
6953         int err;
6954         int drop_inode = 0;
6955
6956         /* do not allow sys_link's with other subvols of the same device */
6957         if (root->root_key.objectid != BTRFS_I(inode)->root->root_key.objectid)
6958                 return -EXDEV;
6959
6960         if (inode->i_nlink >= BTRFS_LINK_MAX)
6961                 return -EMLINK;
6962
6963         err = btrfs_set_inode_index(BTRFS_I(dir), &index);
6964         if (err)
6965                 goto fail;
6966
6967         /*
6968          * 2 items for inode and inode ref
6969          * 2 items for dir items
6970          * 1 item for parent inode
6971          * 1 item for orphan item deletion if O_TMPFILE
6972          */
6973         trans = btrfs_start_transaction(root, inode->i_nlink ? 5 : 6);
6974         if (IS_ERR(trans)) {
6975                 err = PTR_ERR(trans);
6976                 trans = NULL;
6977                 goto fail;
6978         }
6979
6980         /* There are several dir indexes for this inode, clear the cache. */
6981         BTRFS_I(inode)->dir_index = 0ULL;
6982         inc_nlink(inode);
6983         inode_inc_iversion(inode);
6984         inode->i_ctime = current_time(inode);
6985         ihold(inode);
6986         set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
6987
6988         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6989                         1, index);
6990
6991         if (err) {
6992                 drop_inode = 1;
6993         } else {
6994                 struct dentry *parent = dentry->d_parent;
6995
6996                 err = btrfs_update_inode(trans, root, inode);
6997                 if (err)
6998                         goto fail;
6999                 if (inode->i_nlink == 1) {
7000                         /*
7001                          * If new hard link count is 1, it's a file created
7002                          * with open(2) O_TMPFILE flag.
7003                          */
7004                         err = btrfs_orphan_del(trans, BTRFS_I(inode));
7005                         if (err)
7006                                 goto fail;
7007                 }
7008                 d_instantiate(dentry, inode);
7009                 btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent);
7010         }
7011
7012 fail:
7013         if (trans)
7014                 btrfs_end_transaction(trans);
7015         if (drop_inode) {
7016                 inode_dec_link_count(inode);
7017                 iput(inode);
7018         }
7019         btrfs_btree_balance_dirty(fs_info);
7020         return err;
7021 }
7022
7023 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
7024 {
7025         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
7026         struct inode *inode = NULL;
7027         struct btrfs_trans_handle *trans;
7028         struct btrfs_root *root = BTRFS_I(dir)->root;
7029         int err = 0;
7030         u64 objectid = 0;
7031         u64 index = 0;
7032
7033         /*
7034          * 2 items for inode and ref
7035          * 2 items for dir items
7036          * 1 for xattr if selinux is on
7037          */
7038         trans = btrfs_start_transaction(root, 5);
7039         if (IS_ERR(trans))
7040                 return PTR_ERR(trans);
7041
7042         err = btrfs_find_free_objectid(root, &objectid);
7043         if (err)
7044                 goto out_fail;
7045
7046         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
7047                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
7048                         S_IFDIR | mode, &index);
7049         if (IS_ERR(inode)) {
7050                 err = PTR_ERR(inode);
7051                 inode = NULL;
7052                 goto out_fail;
7053         }
7054
7055         /* these must be set before we unlock the inode */
7056         inode->i_op = &btrfs_dir_inode_operations;
7057         inode->i_fop = &btrfs_dir_file_operations;
7058
7059         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
7060         if (err)
7061                 goto out_fail;
7062
7063         btrfs_i_size_write(BTRFS_I(inode), 0);
7064         err = btrfs_update_inode(trans, root, inode);
7065         if (err)
7066                 goto out_fail;
7067
7068         err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
7069                         dentry->d_name.name,
7070                         dentry->d_name.len, 0, index);
7071         if (err)
7072                 goto out_fail;
7073
7074         d_instantiate_new(dentry, inode);
7075
7076 out_fail:
7077         btrfs_end_transaction(trans);
7078         if (err && inode) {
7079                 inode_dec_link_count(inode);
7080                 discard_new_inode(inode);
7081         }
7082         btrfs_btree_balance_dirty(fs_info);
7083         return err;
7084 }
7085
7086 static noinline int uncompress_inline(struct btrfs_path *path,
7087                                       struct page *page,
7088                                       size_t pg_offset, u64 extent_offset,
7089                                       struct btrfs_file_extent_item *item)
7090 {
7091         int ret;
7092         struct extent_buffer *leaf = path->nodes[0];
7093         char *tmp;
7094         size_t max_size;
7095         unsigned long inline_size;
7096         unsigned long ptr;
7097         int compress_type;
7098
7099         WARN_ON(pg_offset != 0);
7100         compress_type = btrfs_file_extent_compression(leaf, item);
7101         max_size = btrfs_file_extent_ram_bytes(leaf, item);
7102         inline_size = btrfs_file_extent_inline_item_len(leaf,
7103                                         btrfs_item_nr(path->slots[0]));
7104         tmp = kmalloc(inline_size, GFP_NOFS);
7105         if (!tmp)
7106                 return -ENOMEM;
7107         ptr = btrfs_file_extent_inline_start(item);
7108
7109         read_extent_buffer(leaf, tmp, ptr, inline_size);
7110
7111         max_size = min_t(unsigned long, PAGE_SIZE, max_size);
7112         ret = btrfs_decompress(compress_type, tmp, page,
7113                                extent_offset, inline_size, max_size);
7114
7115         /*
7116          * decompression code contains a memset to fill in any space between the end
7117          * of the uncompressed data and the end of max_size in case the decompressed
7118          * data ends up shorter than ram_bytes.  That doesn't cover the hole between
7119          * the end of an inline extent and the beginning of the next block, so we
7120          * cover that region here.
7121          */
7122
7123         if (max_size + pg_offset < PAGE_SIZE) {
7124                 char *map = kmap(page);
7125                 memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset);
7126                 kunmap(page);
7127         }
7128         kfree(tmp);
7129         return ret;
7130 }
7131
7132 /*
7133  * a bit scary, this does extent mapping from logical file offset to the disk.
7134  * the ugly parts come from merging extents from the disk with the in-ram
7135  * representation.  This gets more complex because of the data=ordered code,
7136  * where the in-ram extents might be locked pending data=ordered completion.
7137  *
7138  * This also copies inline extents directly into the page.
7139  */
7140 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
7141                                     struct page *page,
7142                                     size_t pg_offset, u64 start, u64 len,
7143                                     int create)
7144 {
7145         struct btrfs_fs_info *fs_info = inode->root->fs_info;
7146         int ret;
7147         int err = 0;
7148         u64 extent_start = 0;
7149         u64 extent_end = 0;
7150         u64 objectid = btrfs_ino(inode);
7151         int extent_type = -1;
7152         struct btrfs_path *path = NULL;
7153         struct btrfs_root *root = inode->root;
7154         struct btrfs_file_extent_item *item;
7155         struct extent_buffer *leaf;
7156         struct btrfs_key found_key;
7157         struct extent_map *em = NULL;
7158         struct extent_map_tree *em_tree = &inode->extent_tree;
7159         struct extent_io_tree *io_tree = &inode->io_tree;
7160         const bool new_inline = !page || create;
7161
7162         read_lock(&em_tree->lock);
7163         em = lookup_extent_mapping(em_tree, start, len);
7164         if (em)
7165                 em->bdev = fs_info->fs_devices->latest_bdev;
7166         read_unlock(&em_tree->lock);
7167
7168         if (em) {
7169                 if (em->start > start || em->start + em->len <= start)
7170                         free_extent_map(em);
7171                 else if (em->block_start == EXTENT_MAP_INLINE && page)
7172                         free_extent_map(em);
7173                 else
7174                         goto out;
7175         }
7176         em = alloc_extent_map();
7177         if (!em) {
7178                 err = -ENOMEM;
7179                 goto out;
7180         }
7181         em->bdev = fs_info->fs_devices->latest_bdev;
7182         em->start = EXTENT_MAP_HOLE;
7183         em->orig_start = EXTENT_MAP_HOLE;
7184         em->len = (u64)-1;
7185         em->block_len = (u64)-1;
7186
7187         path = btrfs_alloc_path();
7188         if (!path) {
7189                 err = -ENOMEM;
7190                 goto out;
7191         }
7192
7193         /* Chances are we'll be called again, so go ahead and do readahead */
7194         path->reada = READA_FORWARD;
7195
7196         /*
7197          * Unless we're going to uncompress the inline extent, no sleep would
7198          * happen.
7199          */
7200         path->leave_spinning = 1;
7201
7202         ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0);
7203         if (ret < 0) {
7204                 err = ret;
7205                 goto out;
7206         } else if (ret > 0) {
7207                 if (path->slots[0] == 0)
7208                         goto not_found;
7209                 path->slots[0]--;
7210         }
7211
7212         leaf = path->nodes[0];
7213         item = btrfs_item_ptr(leaf, path->slots[0],
7214                               struct btrfs_file_extent_item);
7215         btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7216         if (found_key.objectid != objectid ||
7217             found_key.type != BTRFS_EXTENT_DATA_KEY) {
7218                 /*
7219                  * If we backup past the first extent we want to move forward
7220                  * and see if there is an extent in front of us, otherwise we'll
7221                  * say there is a hole for our whole search range which can
7222                  * cause problems.
7223                  */
7224                 extent_end = start;
7225                 goto next;
7226         }
7227
7228         extent_type = btrfs_file_extent_type(leaf, item);
7229         extent_start = found_key.offset;
7230         if (extent_type == BTRFS_FILE_EXTENT_REG ||
7231             extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
7232                 /* Only regular file could have regular/prealloc extent */
7233                 if (!S_ISREG(inode->vfs_inode.i_mode)) {
7234                         err = -EUCLEAN;
7235                         btrfs_crit(fs_info,
7236                 "regular/prealloc extent found for non-regular inode %llu",
7237                                    btrfs_ino(inode));
7238                         goto out;
7239                 }
7240                 extent_end = extent_start +
7241                        btrfs_file_extent_num_bytes(leaf, item);
7242
7243                 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item,
7244                                                        extent_start);
7245         } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
7246                 size_t size;
7247
7248                 size = btrfs_file_extent_ram_bytes(leaf, item);
7249                 extent_end = ALIGN(extent_start + size,
7250                                    fs_info->sectorsize);
7251
7252                 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item,
7253                                                       path->slots[0],
7254                                                       extent_start);
7255         }
7256 next:
7257         if (start >= extent_end) {
7258                 path->slots[0]++;
7259                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
7260                         ret = btrfs_next_leaf(root, path);
7261                         if (ret < 0) {
7262                                 err = ret;
7263                                 goto out;
7264                         } else if (ret > 0) {
7265                                 goto not_found;
7266                         }
7267                         leaf = path->nodes[0];
7268                 }
7269                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7270                 if (found_key.objectid != objectid ||
7271                     found_key.type != BTRFS_EXTENT_DATA_KEY)
7272                         goto not_found;
7273                 if (start + len <= found_key.offset)
7274                         goto not_found;
7275                 if (start > found_key.offset)
7276                         goto next;
7277
7278                 /* New extent overlaps with existing one */
7279                 em->start = start;
7280                 em->orig_start = start;
7281                 em->len = found_key.offset - start;
7282                 em->block_start = EXTENT_MAP_HOLE;
7283                 goto insert;
7284         }
7285
7286         btrfs_extent_item_to_extent_map(inode, path, item,
7287                         new_inline, em);
7288
7289         if (extent_type == BTRFS_FILE_EXTENT_REG ||
7290             extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
7291                 goto insert;
7292         } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
7293                 unsigned long ptr;
7294                 char *map;
7295                 size_t size;
7296                 size_t extent_offset;
7297                 size_t copy_size;
7298
7299                 if (new_inline)
7300                         goto out;
7301
7302                 size = btrfs_file_extent_ram_bytes(leaf, item);
7303                 extent_offset = page_offset(page) + pg_offset - extent_start;
7304                 copy_size = min_t(u64, PAGE_SIZE - pg_offset,
7305                                   size - extent_offset);
7306                 em->start = extent_start + extent_offset;
7307                 em->len = ALIGN(copy_size, fs_info->sectorsize);
7308                 em->orig_block_len = em->len;
7309                 em->orig_start = em->start;
7310                 ptr = btrfs_file_extent_inline_start(item) + extent_offset;
7311
7312                 btrfs_set_path_blocking(path);
7313                 if (!PageUptodate(page)) {
7314                         if (btrfs_file_extent_compression(leaf, item) !=
7315                             BTRFS_COMPRESS_NONE) {
7316                                 ret = uncompress_inline(path, page, pg_offset,
7317                                                         extent_offset, item);
7318                                 if (ret) {
7319                                         err = ret;
7320                                         goto out;
7321                                 }
7322                         } else {
7323                                 map = kmap(page);
7324                                 read_extent_buffer(leaf, map + pg_offset, ptr,
7325                                                    copy_size);
7326                                 if (pg_offset + copy_size < PAGE_SIZE) {
7327                                         memset(map + pg_offset + copy_size, 0,
7328                                                PAGE_SIZE - pg_offset -
7329                                                copy_size);
7330                                 }
7331                                 kunmap(page);
7332                         }
7333                         flush_dcache_page(page);
7334                 }
7335                 set_extent_uptodate(io_tree, em->start,
7336                                     extent_map_end(em) - 1, NULL, GFP_NOFS);
7337                 goto insert;
7338         }
7339 not_found:
7340         em->start = start;
7341         em->orig_start = start;
7342         em->len = len;
7343         em->block_start = EXTENT_MAP_HOLE;
7344 insert:
7345         btrfs_release_path(path);
7346         if (em->start > start || extent_map_end(em) <= start) {
7347                 btrfs_err(fs_info,
7348                           "bad extent! em: [%llu %llu] passed [%llu %llu]",
7349                           em->start, em->len, start, len);
7350                 err = -EIO;
7351                 goto out;
7352         }
7353
7354         err = 0;
7355         write_lock(&em_tree->lock);
7356         err = btrfs_add_extent_mapping(fs_info, em_tree, &em, start, len);
7357         write_unlock(&em_tree->lock);
7358 out:
7359         btrfs_free_path(path);
7360
7361         trace_btrfs_get_extent(root, inode, em);
7362
7363         if (err) {
7364                 free_extent_map(em);
7365                 return ERR_PTR(err);
7366         }
7367         BUG_ON(!em); /* Error is always set */
7368         return em;
7369 }
7370
7371 struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode,
7372                                            u64 start, u64 len)
7373 {
7374         struct extent_map *em;
7375         struct extent_map *hole_em = NULL;
7376         u64 delalloc_start = start;
7377         u64 end;
7378         u64 delalloc_len;
7379         u64 delalloc_end;
7380         int err = 0;
7381
7382         em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
7383         if (IS_ERR(em))
7384                 return em;
7385         /*
7386          * If our em maps to:
7387          * - a hole or
7388          * - a pre-alloc extent,
7389          * there might actually be delalloc bytes behind it.
7390          */
7391         if (em->block_start != EXTENT_MAP_HOLE &&
7392             !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7393                 return em;
7394         else
7395                 hole_em = em;
7396
7397         /* check to see if we've wrapped (len == -1 or similar) */
7398         end = start + len;
7399         if (end < start)
7400                 end = (u64)-1;
7401         else
7402                 end -= 1;
7403
7404         em = NULL;
7405
7406         /* ok, we didn't find anything, lets look for delalloc */
7407         delalloc_len = count_range_bits(&inode->io_tree, &delalloc_start,
7408                                  end, len, EXTENT_DELALLOC, 1);
7409         delalloc_end = delalloc_start + delalloc_len;
7410         if (delalloc_end < delalloc_start)
7411                 delalloc_end = (u64)-1;
7412
7413         /*
7414          * We didn't find anything useful, return the original results from
7415          * get_extent()
7416          */
7417         if (delalloc_start > end || delalloc_end <= start) {
7418                 em = hole_em;
7419                 hole_em = NULL;
7420                 goto out;
7421         }
7422
7423         /*
7424          * Adjust the delalloc_start to make sure it doesn't go backwards from
7425          * the start they passed in
7426          */
7427         delalloc_start = max(start, delalloc_start);
7428         delalloc_len = delalloc_end - delalloc_start;
7429
7430         if (delalloc_len > 0) {
7431                 u64 hole_start;
7432                 u64 hole_len;
7433                 const u64 hole_end = extent_map_end(hole_em);
7434
7435                 em = alloc_extent_map();
7436                 if (!em) {
7437                         err = -ENOMEM;
7438                         goto out;
7439                 }
7440                 em->bdev = NULL;
7441
7442                 ASSERT(hole_em);
7443                 /*
7444                  * When btrfs_get_extent can't find anything it returns one
7445                  * huge hole
7446                  *
7447                  * Make sure what it found really fits our range, and adjust to
7448                  * make sure it is based on the start from the caller
7449                  */
7450                 if (hole_end <= start || hole_em->start > end) {
7451                        free_extent_map(hole_em);
7452                        hole_em = NULL;
7453                 } else {
7454                        hole_start = max(hole_em->start, start);
7455                        hole_len = hole_end - hole_start;
7456                 }
7457
7458                 if (hole_em && delalloc_start > hole_start) {
7459                         /*
7460                          * Our hole starts before our delalloc, so we have to
7461                          * return just the parts of the hole that go until the
7462                          * delalloc starts
7463                          */
7464                         em->len = min(hole_len, delalloc_start - hole_start);
7465                         em->start = hole_start;
7466                         em->orig_start = hole_start;
7467                         /*
7468                          * Don't adjust block start at all, it is fixed at
7469                          * EXTENT_MAP_HOLE
7470                          */
7471                         em->block_start = hole_em->block_start;
7472                         em->block_len = hole_len;
7473                         if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags))
7474                                 set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
7475                 } else {
7476                         /*
7477                          * Hole is out of passed range or it starts after
7478                          * delalloc range
7479                          */
7480                         em->start = delalloc_start;
7481                         em->len = delalloc_len;
7482                         em->orig_start = delalloc_start;
7483                         em->block_start = EXTENT_MAP_DELALLOC;
7484                         em->block_len = delalloc_len;
7485                 }
7486         } else {
7487                 return hole_em;
7488         }
7489 out:
7490
7491         free_extent_map(hole_em);
7492         if (err) {
7493                 free_extent_map(em);
7494                 return ERR_PTR(err);
7495         }
7496         return em;
7497 }
7498
7499 static struct extent_map *btrfs_create_dio_extent(struct inode *inode,
7500                                                   const u64 start,
7501                                                   const u64 len,
7502                                                   const u64 orig_start,
7503                                                   const u64 block_start,
7504                                                   const u64 block_len,
7505                                                   const u64 orig_block_len,
7506                                                   const u64 ram_bytes,
7507                                                   const int type)
7508 {
7509         struct extent_map *em = NULL;
7510         int ret;
7511
7512         if (type != BTRFS_ORDERED_NOCOW) {
7513                 em = create_io_em(inode, start, len, orig_start,
7514                                   block_start, block_len, orig_block_len,
7515                                   ram_bytes,
7516                                   BTRFS_COMPRESS_NONE, /* compress_type */
7517                                   type);
7518                 if (IS_ERR(em))
7519                         goto out;
7520         }
7521         ret = btrfs_add_ordered_extent_dio(inode, start, block_start,
7522                                            len, block_len, type);
7523         if (ret) {
7524                 if (em) {
7525                         free_extent_map(em);
7526                         btrfs_drop_extent_cache(BTRFS_I(inode), start,
7527                                                 start + len - 1, 0);
7528                 }
7529                 em = ERR_PTR(ret);
7530         }
7531  out:
7532
7533         return em;
7534 }
7535
7536 static struct extent_map *btrfs_new_extent_direct(struct inode *inode,
7537                                                   u64 start, u64 len)
7538 {
7539         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7540         struct btrfs_root *root = BTRFS_I(inode)->root;
7541         struct extent_map *em;
7542         struct btrfs_key ins;
7543         u64 alloc_hint;
7544         int ret;
7545
7546         alloc_hint = get_extent_allocation_hint(inode, start, len);
7547         ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize,
7548                                    0, alloc_hint, &ins, 1, 1);
7549         if (ret)
7550                 return ERR_PTR(ret);
7551
7552         em = btrfs_create_dio_extent(inode, start, ins.offset, start,
7553                                      ins.objectid, ins.offset, ins.offset,
7554                                      ins.offset, BTRFS_ORDERED_REGULAR);
7555         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
7556         if (IS_ERR(em))
7557                 btrfs_free_reserved_extent(fs_info, ins.objectid,
7558                                            ins.offset, 1);
7559
7560         return em;
7561 }
7562
7563 /*
7564  * returns 1 when the nocow is safe, < 1 on error, 0 if the
7565  * block must be cow'd
7566  */
7567 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
7568                               u64 *orig_start, u64 *orig_block_len,
7569                               u64 *ram_bytes, bool strict)
7570 {
7571         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7572         struct btrfs_path *path;
7573         int ret;
7574         struct extent_buffer *leaf;
7575         struct btrfs_root *root = BTRFS_I(inode)->root;
7576         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7577         struct btrfs_file_extent_item *fi;
7578         struct btrfs_key key;
7579         u64 disk_bytenr;
7580         u64 backref_offset;
7581         u64 extent_end;
7582         u64 num_bytes;
7583         int slot;
7584         int found_type;
7585         bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW);
7586
7587         path = btrfs_alloc_path();
7588         if (!path)
7589                 return -ENOMEM;
7590
7591         ret = btrfs_lookup_file_extent(NULL, root, path,
7592                         btrfs_ino(BTRFS_I(inode)), offset, 0);
7593         if (ret < 0)
7594                 goto out;
7595
7596         slot = path->slots[0];
7597         if (ret == 1) {
7598                 if (slot == 0) {
7599                         /* can't find the item, must cow */
7600                         ret = 0;
7601                         goto out;
7602                 }
7603                 slot--;
7604         }
7605         ret = 0;
7606         leaf = path->nodes[0];
7607         btrfs_item_key_to_cpu(leaf, &key, slot);
7608         if (key.objectid != btrfs_ino(BTRFS_I(inode)) ||
7609             key.type != BTRFS_EXTENT_DATA_KEY) {
7610                 /* not our file or wrong item type, must cow */
7611                 goto out;
7612         }
7613
7614         if (key.offset > offset) {
7615                 /* Wrong offset, must cow */
7616                 goto out;
7617         }
7618
7619         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
7620         found_type = btrfs_file_extent_type(leaf, fi);
7621         if (found_type != BTRFS_FILE_EXTENT_REG &&
7622             found_type != BTRFS_FILE_EXTENT_PREALLOC) {
7623                 /* not a regular extent, must cow */
7624                 goto out;
7625         }
7626
7627         if (!nocow && found_type == BTRFS_FILE_EXTENT_REG)
7628                 goto out;
7629
7630         extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
7631         if (extent_end <= offset)
7632                 goto out;
7633
7634         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
7635         if (disk_bytenr == 0)
7636                 goto out;
7637
7638         if (btrfs_file_extent_compression(leaf, fi) ||
7639             btrfs_file_extent_encryption(leaf, fi) ||
7640             btrfs_file_extent_other_encoding(leaf, fi))
7641                 goto out;
7642
7643         /*
7644          * Do the same check as in btrfs_cross_ref_exist but without the
7645          * unnecessary search.
7646          */
7647         if (!strict &&
7648             (btrfs_file_extent_generation(leaf, fi) <=
7649              btrfs_root_last_snapshot(&root->root_item)))
7650                 goto out;
7651
7652         backref_offset = btrfs_file_extent_offset(leaf, fi);
7653
7654         if (orig_start) {
7655                 *orig_start = key.offset - backref_offset;
7656                 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
7657                 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
7658         }
7659
7660         if (btrfs_extent_readonly(fs_info, disk_bytenr))
7661                 goto out;
7662
7663         num_bytes = min(offset + *len, extent_end) - offset;
7664         if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7665                 u64 range_end;
7666
7667                 range_end = round_up(offset + num_bytes,
7668                                      root->fs_info->sectorsize) - 1;
7669                 ret = test_range_bit(io_tree, offset, range_end,
7670                                      EXTENT_DELALLOC, 0, NULL);
7671                 if (ret) {
7672                         ret = -EAGAIN;
7673                         goto out;
7674                 }
7675         }
7676
7677         btrfs_release_path(path);
7678
7679         /*
7680          * look for other files referencing this extent, if we
7681          * find any we must cow
7682          */
7683
7684         ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)),
7685                                     key.offset - backref_offset, disk_bytenr,
7686                                     strict);
7687         if (ret) {
7688                 ret = 0;
7689                 goto out;
7690         }
7691
7692         /*
7693          * adjust disk_bytenr and num_bytes to cover just the bytes
7694          * in this extent we are about to write.  If there
7695          * are any csums in that range we have to cow in order
7696          * to keep the csums correct
7697          */
7698         disk_bytenr += backref_offset;
7699         disk_bytenr += offset - key.offset;
7700         if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes))
7701                 goto out;
7702         /*
7703          * all of the above have passed, it is safe to overwrite this extent
7704          * without cow
7705          */
7706         *len = num_bytes;
7707         ret = 1;
7708 out:
7709         btrfs_free_path(path);
7710         return ret;
7711 }
7712
7713 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
7714                               struct extent_state **cached_state, int writing)
7715 {
7716         struct btrfs_ordered_extent *ordered;
7717         int ret = 0;
7718
7719         while (1) {
7720                 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7721                                  cached_state);
7722                 /*
7723                  * We're concerned with the entire range that we're going to be
7724                  * doing DIO to, so we need to make sure there's no ordered
7725                  * extents in this range.
7726                  */
7727                 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,
7728                                                      lockend - lockstart + 1);
7729
7730                 /*
7731                  * We need to make sure there are no buffered pages in this
7732                  * range either, we could have raced between the invalidate in
7733                  * generic_file_direct_write and locking the extent.  The
7734                  * invalidate needs to happen so that reads after a write do not
7735                  * get stale data.
7736                  */
7737                 if (!ordered &&
7738                     (!writing || !filemap_range_has_page(inode->i_mapping,
7739                                                          lockstart, lockend)))
7740                         break;
7741
7742                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7743                                      cached_state);
7744
7745                 if (ordered) {
7746                         /*
7747                          * If we are doing a DIO read and the ordered extent we
7748                          * found is for a buffered write, we can not wait for it
7749                          * to complete and retry, because if we do so we can
7750                          * deadlock with concurrent buffered writes on page
7751                          * locks. This happens only if our DIO read covers more
7752                          * than one extent map, if at this point has already
7753                          * created an ordered extent for a previous extent map
7754                          * and locked its range in the inode's io tree, and a
7755                          * concurrent write against that previous extent map's
7756                          * range and this range started (we unlock the ranges
7757                          * in the io tree only when the bios complete and
7758                          * buffered writes always lock pages before attempting
7759                          * to lock range in the io tree).
7760                          */
7761                         if (writing ||
7762                             test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))
7763                                 btrfs_start_ordered_extent(inode, ordered, 1);
7764                         else
7765                                 ret = -ENOTBLK;
7766                         btrfs_put_ordered_extent(ordered);
7767                 } else {
7768                         /*
7769                          * We could trigger writeback for this range (and wait
7770                          * for it to complete) and then invalidate the pages for
7771                          * this range (through invalidate_inode_pages2_range()),
7772                          * but that can lead us to a deadlock with a concurrent
7773                          * call to readpages() (a buffered read or a defrag call
7774                          * triggered a readahead) on a page lock due to an
7775                          * ordered dio extent we created before but did not have
7776                          * yet a corresponding bio submitted (whence it can not
7777                          * complete), which makes readpages() wait for that
7778                          * ordered extent to complete while holding a lock on
7779                          * that page.
7780                          */
7781                         ret = -ENOTBLK;
7782                 }
7783
7784                 if (ret)
7785                         break;
7786
7787                 cond_resched();
7788         }
7789
7790         return ret;
7791 }
7792
7793 /* The callers of this must take lock_extent() */
7794 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len,
7795                                        u64 orig_start, u64 block_start,
7796                                        u64 block_len, u64 orig_block_len,
7797                                        u64 ram_bytes, int compress_type,
7798                                        int type)
7799 {
7800         struct extent_map_tree *em_tree;
7801         struct extent_map *em;
7802         struct btrfs_root *root = BTRFS_I(inode)->root;
7803         int ret;
7804
7805         ASSERT(type == BTRFS_ORDERED_PREALLOC ||
7806                type == BTRFS_ORDERED_COMPRESSED ||
7807                type == BTRFS_ORDERED_NOCOW ||
7808                type == BTRFS_ORDERED_REGULAR);
7809
7810         em_tree = &BTRFS_I(inode)->extent_tree;
7811         em = alloc_extent_map();
7812         if (!em)
7813                 return ERR_PTR(-ENOMEM);
7814
7815         em->start = start;
7816         em->orig_start = orig_start;
7817         em->len = len;
7818         em->block_len = block_len;
7819         em->block_start = block_start;
7820         em->bdev = root->fs_info->fs_devices->latest_bdev;
7821         em->orig_block_len = orig_block_len;
7822         em->ram_bytes = ram_bytes;
7823         em->generation = -1;
7824         set_bit(EXTENT_FLAG_PINNED, &em->flags);
7825         if (type == BTRFS_ORDERED_PREALLOC) {
7826                 set_bit(EXTENT_FLAG_FILLING, &em->flags);
7827         } else if (type == BTRFS_ORDERED_COMPRESSED) {
7828                 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
7829                 em->compress_type = compress_type;
7830         }
7831
7832         do {
7833                 btrfs_drop_extent_cache(BTRFS_I(inode), em->start,
7834                                 em->start + em->len - 1, 0);
7835                 write_lock(&em_tree->lock);
7836                 ret = add_extent_mapping(em_tree, em, 1);
7837                 write_unlock(&em_tree->lock);
7838                 /*
7839                  * The caller has taken lock_extent(), who could race with us
7840                  * to add em?
7841                  */
7842         } while (ret == -EEXIST);
7843
7844         if (ret) {
7845                 free_extent_map(em);
7846                 return ERR_PTR(ret);
7847         }
7848
7849         /* em got 2 refs now, callers needs to do free_extent_map once. */
7850         return em;
7851 }
7852
7853
7854 static int btrfs_get_blocks_direct_read(struct extent_map *em,
7855                                         struct buffer_head *bh_result,
7856                                         struct inode *inode,
7857                                         u64 start, u64 len)
7858 {
7859         if (em->block_start == EXTENT_MAP_HOLE ||
7860                         test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7861                 return -ENOENT;
7862
7863         len = min(len, em->len - (start - em->start));
7864
7865         bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
7866                 inode->i_blkbits;
7867         bh_result->b_size = len;
7868         bh_result->b_bdev = em->bdev;
7869         set_buffer_mapped(bh_result);
7870
7871         return 0;
7872 }
7873
7874 static int btrfs_get_blocks_direct_write(struct extent_map **map,
7875                                          struct buffer_head *bh_result,
7876                                          struct inode *inode,
7877                                          struct btrfs_dio_data *dio_data,
7878                                          u64 start, u64 len)
7879 {
7880         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7881         struct extent_map *em = *map;
7882         int ret = 0;
7883
7884         /*
7885          * We don't allocate a new extent in the following cases
7886          *
7887          * 1) The inode is marked as NODATACOW. In this case we'll just use the
7888          * existing extent.
7889          * 2) The extent is marked as PREALLOC. We're good to go here and can
7890          * just use the extent.
7891          *
7892          */
7893         if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) ||
7894             ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
7895              em->block_start != EXTENT_MAP_HOLE)) {
7896                 int type;
7897                 u64 block_start, orig_start, orig_block_len, ram_bytes;
7898
7899                 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7900                         type = BTRFS_ORDERED_PREALLOC;
7901                 else
7902                         type = BTRFS_ORDERED_NOCOW;
7903                 len = min(len, em->len - (start - em->start));
7904                 block_start = em->block_start + (start - em->start);
7905
7906                 if (can_nocow_extent(inode, start, &len, &orig_start,
7907                                      &orig_block_len, &ram_bytes, false) == 1 &&
7908                     btrfs_inc_nocow_writers(fs_info, block_start)) {
7909                         struct extent_map *em2;
7910
7911                         em2 = btrfs_create_dio_extent(inode, start, len,
7912                                                       orig_start, block_start,
7913                                                       len, orig_block_len,
7914                                                       ram_bytes, type);
7915                         btrfs_dec_nocow_writers(fs_info, block_start);
7916                         if (type == BTRFS_ORDERED_PREALLOC) {
7917                                 free_extent_map(em);
7918                                 *map = em = em2;
7919                         }
7920
7921                         if (em2 && IS_ERR(em2)) {
7922                                 ret = PTR_ERR(em2);
7923                                 goto out;
7924                         }
7925                         /*
7926                          * For inode marked NODATACOW or extent marked PREALLOC,
7927                          * use the existing or preallocated extent, so does not
7928                          * need to adjust btrfs_space_info's bytes_may_use.
7929                          */
7930                         btrfs_free_reserved_data_space_noquota(inode, start,
7931                                                                len);
7932                         goto skip_cow;
7933                 }
7934         }
7935
7936         /* this will cow the extent */
7937         len = bh_result->b_size;
7938         free_extent_map(em);
7939         *map = em = btrfs_new_extent_direct(inode, start, len);
7940         if (IS_ERR(em)) {
7941                 ret = PTR_ERR(em);
7942                 goto out;
7943         }
7944
7945         len = min(len, em->len - (start - em->start));
7946
7947 skip_cow:
7948         bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
7949                 inode->i_blkbits;
7950         bh_result->b_size = len;
7951         bh_result->b_bdev = em->bdev;
7952         set_buffer_mapped(bh_result);
7953
7954         if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7955                 set_buffer_new(bh_result);
7956
7957         /*
7958          * Need to update the i_size under the extent lock so buffered
7959          * readers will get the updated i_size when we unlock.
7960          */
7961         if (!dio_data->overwrite && start + len > i_size_read(inode))
7962                 i_size_write(inode, start + len);
7963
7964         WARN_ON(dio_data->reserve < len);
7965         dio_data->reserve -= len;
7966         dio_data->unsubmitted_oe_range_end = start + len;
7967         current->journal_info = dio_data;
7968 out:
7969         return ret;
7970 }
7971
7972 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock,
7973                                    struct buffer_head *bh_result, int create)
7974 {
7975         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7976         struct extent_map *em;
7977         struct extent_state *cached_state = NULL;
7978         struct btrfs_dio_data *dio_data = NULL;
7979         u64 start = iblock << inode->i_blkbits;
7980         u64 lockstart, lockend;
7981         u64 len = bh_result->b_size;
7982         int ret = 0;
7983
7984         if (!create)
7985                 len = min_t(u64, len, fs_info->sectorsize);
7986
7987         lockstart = start;
7988         lockend = start + len - 1;
7989
7990         if (current->journal_info) {
7991                 /*
7992                  * Need to pull our outstanding extents and set journal_info to NULL so
7993                  * that anything that needs to check if there's a transaction doesn't get
7994                  * confused.
7995                  */
7996                 dio_data = current->journal_info;
7997                 current->journal_info = NULL;
7998         }
7999
8000         /*
8001          * If this errors out it's because we couldn't invalidate pagecache for
8002          * this range and we need to fallback to buffered.
8003          */
8004         if (lock_extent_direct(inode, lockstart, lockend, &cached_state,
8005                                create)) {
8006                 ret = -ENOTBLK;
8007                 goto err;
8008         }
8009
8010         em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len, 0);
8011         if (IS_ERR(em)) {
8012                 ret = PTR_ERR(em);
8013                 goto unlock_err;
8014         }
8015
8016         /*
8017          * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
8018          * io.  INLINE is special, and we could probably kludge it in here, but
8019          * it's still buffered so for safety lets just fall back to the generic
8020          * buffered path.
8021          *
8022          * For COMPRESSED we _have_ to read the entire extent in so we can
8023          * decompress it, so there will be buffering required no matter what we
8024          * do, so go ahead and fallback to buffered.
8025          *
8026          * We return -ENOTBLK because that's what makes DIO go ahead and go back
8027          * to buffered IO.  Don't blame me, this is the price we pay for using
8028          * the generic code.
8029          */
8030         if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) ||
8031             em->block_start == EXTENT_MAP_INLINE) {
8032                 free_extent_map(em);
8033                 ret = -ENOTBLK;
8034                 goto unlock_err;
8035         }
8036
8037         if (create) {
8038                 ret = btrfs_get_blocks_direct_write(&em, bh_result, inode,
8039                                                     dio_data, start, len);
8040                 if (ret < 0)
8041                         goto unlock_err;
8042
8043                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
8044                                      lockend, &cached_state);
8045         } else {
8046                 ret = btrfs_get_blocks_direct_read(em, bh_result, inode,
8047                                                    start, len);
8048                 /* Can be negative only if we read from a hole */
8049                 if (ret < 0) {
8050                         ret = 0;
8051                         free_extent_map(em);
8052                         goto unlock_err;
8053                 }
8054                 /*
8055                  * We need to unlock only the end area that we aren't using.
8056                  * The rest is going to be unlocked by the endio routine.
8057                  */
8058                 lockstart = start + bh_result->b_size;
8059                 if (lockstart < lockend) {
8060                         unlock_extent_cached(&BTRFS_I(inode)->io_tree,
8061                                              lockstart, lockend, &cached_state);
8062                 } else {
8063                         free_extent_state(cached_state);
8064                 }
8065         }
8066
8067         free_extent_map(em);
8068
8069         return 0;
8070
8071 unlock_err:
8072         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
8073                              &cached_state);
8074 err:
8075         if (dio_data)
8076                 current->journal_info = dio_data;
8077         return ret;
8078 }
8079
8080 static inline blk_status_t submit_dio_repair_bio(struct inode *inode,
8081                                                  struct bio *bio,
8082                                                  int mirror_num)
8083 {
8084         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8085         blk_status_t ret;
8086
8087         BUG_ON(bio_op(bio) == REQ_OP_WRITE);
8088
8089         ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR);
8090         if (ret)
8091                 return ret;
8092
8093         ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
8094
8095         return ret;
8096 }
8097
8098 static int btrfs_check_dio_repairable(struct inode *inode,
8099                                       struct bio *failed_bio,
8100                                       struct io_failure_record *failrec,
8101                                       int failed_mirror)
8102 {
8103         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8104         int num_copies;
8105
8106         num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
8107         if (num_copies == 1) {
8108                 /*
8109                  * we only have a single copy of the data, so don't bother with
8110                  * all the retry and error correction code that follows. no
8111                  * matter what the error is, it is very likely to persist.
8112                  */
8113                 btrfs_debug(fs_info,
8114                         "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
8115                         num_copies, failrec->this_mirror, failed_mirror);
8116                 return 0;
8117         }
8118
8119         failrec->failed_mirror = failed_mirror;
8120         failrec->this_mirror++;
8121         if (failrec->this_mirror == failed_mirror)
8122                 failrec->this_mirror++;
8123
8124         if (failrec->this_mirror > num_copies) {
8125                 btrfs_debug(fs_info,
8126                         "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
8127                         num_copies, failrec->this_mirror, failed_mirror);
8128                 return 0;
8129         }
8130
8131         return 1;
8132 }
8133
8134 static blk_status_t dio_read_error(struct inode *inode, struct bio *failed_bio,
8135                                    struct page *page, unsigned int pgoff,
8136                                    u64 start, u64 end, int failed_mirror,
8137                                    bio_end_io_t *repair_endio, void *repair_arg)
8138 {
8139         struct io_failure_record *failrec;
8140         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
8141         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
8142         struct bio *bio;
8143         int isector;
8144         unsigned int read_mode = 0;
8145         int segs;
8146         int ret;
8147         blk_status_t status;
8148         struct bio_vec bvec;
8149
8150         BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
8151
8152         ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
8153         if (ret)
8154                 return errno_to_blk_status(ret);
8155
8156         ret = btrfs_check_dio_repairable(inode, failed_bio, failrec,
8157                                          failed_mirror);
8158         if (!ret) {
8159                 free_io_failure(failure_tree, io_tree, failrec);
8160                 return BLK_STS_IOERR;
8161         }
8162
8163         segs = bio_segments(failed_bio);
8164         bio_get_first_bvec(failed_bio, &bvec);
8165         if (segs > 1 ||
8166             (bvec.bv_len > btrfs_inode_sectorsize(inode)))
8167                 read_mode |= REQ_FAILFAST_DEV;
8168
8169         isector = start - btrfs_io_bio(failed_bio)->logical;
8170         isector >>= inode->i_sb->s_blocksize_bits;
8171         bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
8172                                 pgoff, isector, repair_endio, repair_arg);
8173         bio->bi_opf = REQ_OP_READ | read_mode;
8174
8175         btrfs_debug(BTRFS_I(inode)->root->fs_info,
8176                     "repair DIO read error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d",
8177                     read_mode, failrec->this_mirror, failrec->in_validation);
8178
8179         status = submit_dio_repair_bio(inode, bio, failrec->this_mirror);
8180         if (status) {
8181                 free_io_failure(failure_tree, io_tree, failrec);
8182                 bio_put(bio);
8183         }
8184
8185         return status;
8186 }
8187
8188 struct btrfs_retry_complete {
8189         struct completion done;
8190         struct inode *inode;
8191         u64 start;
8192         int uptodate;
8193 };
8194
8195 static void btrfs_retry_endio_nocsum(struct bio *bio)
8196 {
8197         struct btrfs_retry_complete *done = bio->bi_private;
8198         struct inode *inode = done->inode;
8199         struct bio_vec *bvec;
8200         struct extent_io_tree *io_tree, *failure_tree;
8201         struct bvec_iter_all iter_all;
8202
8203         if (bio->bi_status)
8204                 goto end;
8205
8206         ASSERT(bio->bi_vcnt == 1);
8207         io_tree = &BTRFS_I(inode)->io_tree;
8208         failure_tree = &BTRFS_I(inode)->io_failure_tree;
8209         ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(inode));
8210
8211         done->uptodate = 1;
8212         ASSERT(!bio_flagged(bio, BIO_CLONED));
8213         bio_for_each_segment_all(bvec, bio, iter_all)
8214                 clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree,
8215                                  io_tree, done->start, bvec->bv_page,
8216                                  btrfs_ino(BTRFS_I(inode)), 0);
8217 end:
8218         complete(&done->done);
8219         bio_put(bio);
8220 }
8221
8222 static blk_status_t __btrfs_correct_data_nocsum(struct inode *inode,
8223                                                 struct btrfs_io_bio *io_bio)
8224 {
8225         struct btrfs_fs_info *fs_info;
8226         struct bio_vec bvec;
8227         struct bvec_iter iter;
8228         struct btrfs_retry_complete done;
8229         u64 start;
8230         unsigned int pgoff;
8231         u32 sectorsize;
8232         int nr_sectors;
8233         blk_status_t ret;
8234         blk_status_t err = BLK_STS_OK;
8235
8236         fs_info = BTRFS_I(inode)->root->fs_info;
8237         sectorsize = fs_info->sectorsize;
8238
8239         start = io_bio->logical;
8240         done.inode = inode;
8241         io_bio->bio.bi_iter = io_bio->iter;
8242
8243         bio_for_each_segment(bvec, &io_bio->bio, iter) {
8244                 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
8245                 pgoff = bvec.bv_offset;
8246
8247 next_block_or_try_again:
8248                 done.uptodate = 0;
8249                 done.start = start;
8250                 init_completion(&done.done);
8251
8252                 ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
8253                                 pgoff, start, start + sectorsize - 1,
8254                                 io_bio->mirror_num,
8255                                 btrfs_retry_endio_nocsum, &done);
8256                 if (ret) {
8257                         err = ret;
8258                         goto next;
8259                 }
8260
8261                 wait_for_completion_io(&done.done);
8262
8263                 if (!done.uptodate) {
8264                         /* We might have another mirror, so try again */
8265                         goto next_block_or_try_again;
8266                 }
8267
8268 next:
8269                 start += sectorsize;
8270
8271                 nr_sectors--;
8272                 if (nr_sectors) {
8273                         pgoff += sectorsize;
8274                         ASSERT(pgoff < PAGE_SIZE);
8275                         goto next_block_or_try_again;
8276                 }
8277         }
8278
8279         return err;
8280 }
8281
8282 static void btrfs_retry_endio(struct bio *bio)
8283 {
8284         struct btrfs_retry_complete *done = bio->bi_private;
8285         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8286         struct extent_io_tree *io_tree, *failure_tree;
8287         struct inode *inode = done->inode;
8288         struct bio_vec *bvec;
8289         int uptodate;
8290         int ret;
8291         int i = 0;
8292         struct bvec_iter_all iter_all;
8293
8294         if (bio->bi_status)
8295                 goto end;
8296
8297         uptodate = 1;
8298
8299         ASSERT(bio->bi_vcnt == 1);
8300         ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(done->inode));
8301
8302         io_tree = &BTRFS_I(inode)->io_tree;
8303         failure_tree = &BTRFS_I(inode)->io_failure_tree;
8304
8305         ASSERT(!bio_flagged(bio, BIO_CLONED));
8306         bio_for_each_segment_all(bvec, bio, iter_all) {
8307                 ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page,
8308                                              bvec->bv_offset, done->start,
8309                                              bvec->bv_len);
8310                 if (!ret)
8311                         clean_io_failure(BTRFS_I(inode)->root->fs_info,
8312                                          failure_tree, io_tree, done->start,
8313                                          bvec->bv_page,
8314                                          btrfs_ino(BTRFS_I(inode)),
8315                                          bvec->bv_offset);
8316                 else
8317                         uptodate = 0;
8318                 i++;
8319         }
8320
8321         done->uptodate = uptodate;
8322 end:
8323         complete(&done->done);
8324         bio_put(bio);
8325 }
8326
8327 static blk_status_t __btrfs_subio_endio_read(struct inode *inode,
8328                 struct btrfs_io_bio *io_bio, blk_status_t err)
8329 {
8330         struct btrfs_fs_info *fs_info;
8331         struct bio_vec bvec;
8332         struct bvec_iter iter;
8333         struct btrfs_retry_complete done;
8334         u64 start;
8335         u64 offset = 0;
8336         u32 sectorsize;
8337         int nr_sectors;
8338         unsigned int pgoff;
8339         int csum_pos;
8340         bool uptodate = (err == 0);
8341         int ret;
8342         blk_status_t status;
8343
8344         fs_info = BTRFS_I(inode)->root->fs_info;
8345         sectorsize = fs_info->sectorsize;
8346
8347         err = BLK_STS_OK;
8348         start = io_bio->logical;
8349         done.inode = inode;
8350         io_bio->bio.bi_iter = io_bio->iter;
8351
8352         bio_for_each_segment(bvec, &io_bio->bio, iter) {
8353                 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
8354
8355                 pgoff = bvec.bv_offset;
8356 next_block:
8357                 if (uptodate) {
8358                         csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset);
8359                         ret = __readpage_endio_check(inode, io_bio, csum_pos,
8360                                         bvec.bv_page, pgoff, start, sectorsize);
8361                         if (likely(!ret))
8362                                 goto next;
8363                 }
8364 try_again:
8365                 done.uptodate = 0;
8366                 done.start = start;
8367                 init_completion(&done.done);
8368
8369                 status = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
8370                                         pgoff, start, start + sectorsize - 1,
8371                                         io_bio->mirror_num, btrfs_retry_endio,
8372                                         &done);
8373                 if (status) {
8374                         err = status;
8375                         goto next;
8376                 }
8377
8378                 wait_for_completion_io(&done.done);
8379
8380                 if (!done.uptodate) {
8381                         /* We might have another mirror, so try again */
8382                         goto try_again;
8383                 }
8384 next:
8385                 offset += sectorsize;
8386                 start += sectorsize;
8387
8388                 ASSERT(nr_sectors);
8389
8390                 nr_sectors--;
8391                 if (nr_sectors) {
8392                         pgoff += sectorsize;
8393                         ASSERT(pgoff < PAGE_SIZE);
8394                         goto next_block;
8395                 }
8396         }
8397
8398         return err;
8399 }
8400
8401 static blk_status_t btrfs_subio_endio_read(struct inode *inode,
8402                 struct btrfs_io_bio *io_bio, blk_status_t err)
8403 {
8404         bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
8405
8406         if (skip_csum) {
8407                 if (unlikely(err))
8408                         return __btrfs_correct_data_nocsum(inode, io_bio);
8409                 else
8410                         return BLK_STS_OK;
8411         } else {
8412                 return __btrfs_subio_endio_read(inode, io_bio, err);
8413         }
8414 }
8415
8416 static void btrfs_endio_direct_read(struct bio *bio)
8417 {
8418         struct btrfs_dio_private *dip = bio->bi_private;
8419         struct inode *inode = dip->inode;
8420         struct bio *dio_bio;
8421         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8422         blk_status_t err = bio->bi_status;
8423
8424         if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED)
8425                 err = btrfs_subio_endio_read(inode, io_bio, err);
8426
8427         unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset,
8428                       dip->logical_offset + dip->bytes - 1);
8429         dio_bio = dip->dio_bio;
8430
8431         kfree(dip);
8432
8433         dio_bio->bi_status = err;
8434         dio_end_io(dio_bio);
8435         btrfs_io_bio_free_csum(io_bio);
8436         bio_put(bio);
8437 }
8438
8439 static void __endio_write_update_ordered(struct inode *inode,
8440                                          const u64 offset, const u64 bytes,
8441                                          const bool uptodate)
8442 {
8443         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8444         struct btrfs_ordered_extent *ordered = NULL;
8445         struct btrfs_workqueue *wq;
8446         u64 ordered_offset = offset;
8447         u64 ordered_bytes = bytes;
8448         u64 last_offset;
8449
8450         if (btrfs_is_free_space_inode(BTRFS_I(inode)))
8451                 wq = fs_info->endio_freespace_worker;
8452         else
8453                 wq = fs_info->endio_write_workers;
8454
8455         while (ordered_offset < offset + bytes) {
8456                 last_offset = ordered_offset;
8457                 if (btrfs_dec_test_first_ordered_pending(inode, &ordered,
8458                                                            &ordered_offset,
8459                                                            ordered_bytes,
8460                                                            uptodate)) {
8461                         btrfs_init_work(&ordered->work, finish_ordered_fn, NULL,
8462                                         NULL);
8463                         btrfs_queue_work(wq, &ordered->work);
8464                 }
8465                 /*
8466                  * If btrfs_dec_test_ordered_pending does not find any ordered
8467                  * extent in the range, we can exit.
8468                  */
8469                 if (ordered_offset == last_offset)
8470                         return;
8471                 /*
8472                  * Our bio might span multiple ordered extents. In this case
8473                  * we keep going until we have accounted the whole dio.
8474                  */
8475                 if (ordered_offset < offset + bytes) {
8476                         ordered_bytes = offset + bytes - ordered_offset;
8477                         ordered = NULL;
8478                 }
8479         }
8480 }
8481
8482 static void btrfs_endio_direct_write(struct bio *bio)
8483 {
8484         struct btrfs_dio_private *dip = bio->bi_private;
8485         struct bio *dio_bio = dip->dio_bio;
8486
8487         __endio_write_update_ordered(dip->inode, dip->logical_offset,
8488                                      dip->bytes, !bio->bi_status);
8489
8490         kfree(dip);
8491
8492         dio_bio->bi_status = bio->bi_status;
8493         dio_end_io(dio_bio);
8494         bio_put(bio);
8495 }
8496
8497 static blk_status_t btrfs_submit_bio_start_direct_io(void *private_data,
8498                                     struct bio *bio, u64 offset)
8499 {
8500         struct inode *inode = private_data;
8501         blk_status_t ret;
8502         ret = btrfs_csum_one_bio(inode, bio, offset, 1);
8503         BUG_ON(ret); /* -ENOMEM */
8504         return 0;
8505 }
8506
8507 static void btrfs_end_dio_bio(struct bio *bio)
8508 {
8509         struct btrfs_dio_private *dip = bio->bi_private;
8510         blk_status_t err = bio->bi_status;
8511
8512         if (err)
8513                 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info,
8514                            "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d",
8515                            btrfs_ino(BTRFS_I(dip->inode)), bio_op(bio),
8516                            bio->bi_opf,
8517                            (unsigned long long)bio->bi_iter.bi_sector,
8518                            bio->bi_iter.bi_size, err);
8519
8520         if (dip->subio_endio)
8521                 err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err);
8522
8523         if (err) {
8524                 /*
8525                  * We want to perceive the errors flag being set before
8526                  * decrementing the reference count. We don't need a barrier
8527                  * since atomic operations with a return value are fully
8528                  * ordered as per atomic_t.txt
8529                  */
8530                 dip->errors = 1;
8531         }
8532
8533         /* if there are more bios still pending for this dio, just exit */
8534         if (!atomic_dec_and_test(&dip->pending_bios))
8535                 goto out;
8536
8537         if (dip->errors) {
8538                 bio_io_error(dip->orig_bio);
8539         } else {
8540                 dip->dio_bio->bi_status = BLK_STS_OK;
8541                 bio_endio(dip->orig_bio);
8542         }
8543 out:
8544         bio_put(bio);
8545 }
8546
8547 static inline blk_status_t btrfs_lookup_and_bind_dio_csum(struct inode *inode,
8548                                                  struct btrfs_dio_private *dip,
8549                                                  struct bio *bio,
8550                                                  u64 file_offset)
8551 {
8552         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8553         struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio);
8554         u16 csum_size;
8555         blk_status_t ret;
8556
8557         /*
8558          * We load all the csum data we need when we submit
8559          * the first bio to reduce the csum tree search and
8560          * contention.
8561          */
8562         if (dip->logical_offset == file_offset) {
8563                 ret = btrfs_lookup_bio_sums_dio(inode, dip->orig_bio,
8564                                                 file_offset);
8565                 if (ret)
8566                         return ret;
8567         }
8568
8569         if (bio == dip->orig_bio)
8570                 return 0;
8571
8572         file_offset -= dip->logical_offset;
8573         file_offset >>= inode->i_sb->s_blocksize_bits;
8574         csum_size = btrfs_super_csum_size(btrfs_sb(inode->i_sb)->super_copy);
8575         io_bio->csum = orig_io_bio->csum + csum_size * file_offset;
8576
8577         return 0;
8578 }
8579
8580 static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio,
8581                 struct inode *inode, u64 file_offset, int async_submit)
8582 {
8583         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8584         struct btrfs_dio_private *dip = bio->bi_private;
8585         bool write = bio_op(bio) == REQ_OP_WRITE;
8586         blk_status_t ret;
8587
8588         /* Check btrfs_submit_bio_hook() for rules about async submit. */
8589         if (async_submit)
8590                 async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers);
8591
8592         if (!write) {
8593                 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
8594                 if (ret)
8595                         goto err;
8596         }
8597
8598         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
8599                 goto map;
8600
8601         if (write && async_submit) {
8602                 ret = btrfs_wq_submit_bio(fs_info, bio, 0, 0,
8603                                           file_offset, inode,
8604                                           btrfs_submit_bio_start_direct_io);
8605                 goto err;
8606         } else if (write) {
8607                 /*
8608                  * If we aren't doing async submit, calculate the csum of the
8609                  * bio now.
8610                  */
8611                 ret = btrfs_csum_one_bio(inode, bio, file_offset, 1);
8612                 if (ret)
8613                         goto err;
8614         } else {
8615                 ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio,
8616                                                      file_offset);
8617                 if (ret)
8618                         goto err;
8619         }
8620 map:
8621         ret = btrfs_map_bio(fs_info, bio, 0, 0);
8622 err:
8623         return ret;
8624 }
8625
8626 /*
8627  * If this succeeds, the btrfs_dio_private is responsible for cleaning up locked
8628  * or ordered extents whether or not we submit any bios.
8629  */
8630 static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio,
8631                                                           struct inode *inode,
8632                                                           loff_t file_offset)
8633 {
8634         const bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
8635         struct btrfs_dio_private *dip;
8636         struct bio *bio;
8637
8638         dip = kzalloc(sizeof(*dip), GFP_NOFS);
8639         if (!dip)
8640                 return NULL;
8641
8642         bio = btrfs_bio_clone(dio_bio);
8643         bio->bi_private = dip;
8644         btrfs_io_bio(bio)->logical = file_offset;
8645
8646         dip->private = dio_bio->bi_private;
8647         dip->inode = inode;
8648         dip->logical_offset = file_offset;
8649         dip->bytes = dio_bio->bi_iter.bi_size;
8650         dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
8651         dip->orig_bio = bio;
8652         dip->dio_bio = dio_bio;
8653         atomic_set(&dip->pending_bios, 1);
8654
8655         if (write) {
8656                 struct btrfs_dio_data *dio_data = current->journal_info;
8657
8658                 /*
8659                  * Setting range start and end to the same value means that
8660                  * no cleanup will happen in btrfs_direct_IO
8661                  */
8662                 dio_data->unsubmitted_oe_range_end = dip->logical_offset +
8663                         dip->bytes;
8664                 dio_data->unsubmitted_oe_range_start =
8665                         dio_data->unsubmitted_oe_range_end;
8666
8667                 bio->bi_end_io = btrfs_endio_direct_write;
8668         } else {
8669                 bio->bi_end_io = btrfs_endio_direct_read;
8670                 dip->subio_endio = btrfs_subio_endio_read;
8671         }
8672         return dip;
8673 }
8674
8675 static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode,
8676                                 loff_t file_offset)
8677 {
8678         const bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
8679         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8680         struct btrfs_dio_private *dip;
8681         struct bio *bio;
8682         struct bio *orig_bio;
8683         u64 start_sector;
8684         int async_submit = 0;
8685         u64 submit_len;
8686         int clone_offset = 0;
8687         int clone_len;
8688         int ret;
8689         blk_status_t status;
8690         struct btrfs_io_geometry geom;
8691
8692         dip = btrfs_create_dio_private(dio_bio, inode, file_offset);
8693         if (!dip) {
8694                 if (!write) {
8695                         unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
8696                                 file_offset + dio_bio->bi_iter.bi_size - 1);
8697                 }
8698                 dio_bio->bi_status = BLK_STS_RESOURCE;
8699                 dio_end_io(dio_bio);
8700                 return;
8701         }
8702
8703         orig_bio = dip->orig_bio;
8704         start_sector = orig_bio->bi_iter.bi_sector;
8705         submit_len = orig_bio->bi_iter.bi_size;
8706         ret = btrfs_get_io_geometry(fs_info, btrfs_op(orig_bio),
8707                                     start_sector << 9, submit_len, &geom);
8708         if (ret)
8709                 goto out_err;
8710
8711         if (geom.len >= submit_len) {
8712                 bio = orig_bio;
8713                 dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED;
8714                 goto submit;
8715         }
8716
8717         /* async crcs make it difficult to collect full stripe writes. */
8718         if (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK)
8719                 async_submit = 0;
8720         else
8721                 async_submit = 1;
8722
8723         /* bio split */
8724         ASSERT(geom.len <= INT_MAX);
8725         do {
8726                 clone_len = min_t(int, submit_len, geom.len);
8727
8728                 /*
8729                  * This will never fail as it's passing GPF_NOFS and
8730                  * the allocation is backed by btrfs_bioset.
8731                  */
8732                 bio = btrfs_bio_clone_partial(orig_bio, clone_offset,
8733                                               clone_len);
8734                 bio->bi_private = dip;
8735                 bio->bi_end_io = btrfs_end_dio_bio;
8736                 btrfs_io_bio(bio)->logical = file_offset;
8737
8738                 ASSERT(submit_len >= clone_len);
8739                 submit_len -= clone_len;
8740                 if (submit_len == 0)
8741                         break;
8742
8743                 /*
8744                  * Increase the count before we submit the bio so we know
8745                  * the end IO handler won't happen before we increase the
8746                  * count. Otherwise, the dip might get freed before we're
8747                  * done setting it up.
8748                  */
8749                 atomic_inc(&dip->pending_bios);
8750
8751                 status = btrfs_submit_dio_bio(bio, inode, file_offset,
8752                                                 async_submit);
8753                 if (status) {
8754                         bio_put(bio);
8755                         atomic_dec(&dip->pending_bios);
8756                         goto out_err;
8757                 }
8758
8759                 clone_offset += clone_len;
8760                 start_sector += clone_len >> 9;
8761                 file_offset += clone_len;
8762
8763                 ret = btrfs_get_io_geometry(fs_info, btrfs_op(orig_bio),
8764                                       start_sector << 9, submit_len, &geom);
8765                 if (ret)
8766                         goto out_err;
8767         } while (submit_len > 0);
8768
8769 submit:
8770         status = btrfs_submit_dio_bio(bio, inode, file_offset, async_submit);
8771         if (!status)
8772                 return;
8773
8774         if (bio != orig_bio)
8775                 bio_put(bio);
8776 out_err:
8777         dip->errors = 1;
8778         /*
8779          * Before atomic variable goto zero, we must  make sure dip->errors is
8780          * perceived to be set. This ordering is ensured by the fact that an
8781          * atomic operations with a return value are fully ordered as per
8782          * atomic_t.txt
8783          */
8784         if (atomic_dec_and_test(&dip->pending_bios))
8785                 bio_io_error(dip->orig_bio);
8786 }
8787
8788 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
8789                                const struct iov_iter *iter, loff_t offset)
8790 {
8791         int seg;
8792         int i;
8793         unsigned int blocksize_mask = fs_info->sectorsize - 1;
8794         ssize_t retval = -EINVAL;
8795
8796         if (offset & blocksize_mask)
8797                 goto out;
8798
8799         if (iov_iter_alignment(iter) & blocksize_mask)
8800                 goto out;
8801
8802         /* If this is a write we don't need to check anymore */
8803         if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter))
8804                 return 0;
8805         /*
8806          * Check to make sure we don't have duplicate iov_base's in this
8807          * iovec, if so return EINVAL, otherwise we'll get csum errors
8808          * when reading back.
8809          */
8810         for (seg = 0; seg < iter->nr_segs; seg++) {
8811                 for (i = seg + 1; i < iter->nr_segs; i++) {
8812                         if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
8813                                 goto out;
8814                 }
8815         }
8816         retval = 0;
8817 out:
8818         return retval;
8819 }
8820
8821 static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
8822 {
8823         struct file *file = iocb->ki_filp;
8824         struct inode *inode = file->f_mapping->host;
8825         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8826         struct btrfs_dio_data dio_data = { 0 };
8827         struct extent_changeset *data_reserved = NULL;
8828         loff_t offset = iocb->ki_pos;
8829         size_t count = 0;
8830         int flags = 0;
8831         bool wakeup = true;
8832         bool relock = false;
8833         ssize_t ret;
8834
8835         if (check_direct_IO(fs_info, iter, offset))
8836                 return 0;
8837
8838         inode_dio_begin(inode);
8839
8840         /*
8841          * The generic stuff only does filemap_write_and_wait_range, which
8842          * isn't enough if we've written compressed pages to this area, so
8843          * we need to flush the dirty pages again to make absolutely sure
8844          * that any outstanding dirty pages are on disk.
8845          */
8846         count = iov_iter_count(iter);
8847         if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
8848                      &BTRFS_I(inode)->runtime_flags))
8849                 filemap_fdatawrite_range(inode->i_mapping, offset,
8850                                          offset + count - 1);
8851
8852         if (iov_iter_rw(iter) == WRITE) {
8853                 /*
8854                  * If the write DIO is beyond the EOF, we need update
8855                  * the isize, but it is protected by i_mutex. So we can
8856                  * not unlock the i_mutex at this case.
8857                  */
8858                 if (offset + count <= inode->i_size) {
8859                         dio_data.overwrite = 1;
8860                         inode_unlock(inode);
8861                         relock = true;
8862                 }
8863                 ret = btrfs_delalloc_reserve_space(inode, &data_reserved,
8864                                                    offset, count);
8865                 if (ret)
8866                         goto out;
8867
8868                 /*
8869                  * We need to know how many extents we reserved so that we can
8870                  * do the accounting properly if we go over the number we
8871                  * originally calculated.  Abuse current->journal_info for this.
8872                  */
8873                 dio_data.reserve = round_up(count,
8874                                             fs_info->sectorsize);
8875                 dio_data.unsubmitted_oe_range_start = (u64)offset;
8876                 dio_data.unsubmitted_oe_range_end = (u64)offset;
8877                 current->journal_info = &dio_data;
8878                 down_read(&BTRFS_I(inode)->dio_sem);
8879         } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
8880                                      &BTRFS_I(inode)->runtime_flags)) {
8881                 inode_dio_end(inode);
8882                 flags = DIO_LOCKING | DIO_SKIP_HOLES;
8883                 wakeup = false;
8884         }
8885
8886         ret = __blockdev_direct_IO(iocb, inode,
8887                                    fs_info->fs_devices->latest_bdev,
8888                                    iter, btrfs_get_blocks_direct, NULL,
8889                                    btrfs_submit_direct, flags);
8890         if (iov_iter_rw(iter) == WRITE) {
8891                 up_read(&BTRFS_I(inode)->dio_sem);
8892                 current->journal_info = NULL;
8893                 if (ret < 0 && ret != -EIOCBQUEUED) {
8894                         if (dio_data.reserve)
8895                                 btrfs_delalloc_release_space(inode, data_reserved,
8896                                         offset, dio_data.reserve, true);
8897                         /*
8898                          * On error we might have left some ordered extents
8899                          * without submitting corresponding bios for them, so
8900                          * cleanup them up to avoid other tasks getting them
8901                          * and waiting for them to complete forever.
8902                          */
8903                         if (dio_data.unsubmitted_oe_range_start <
8904                             dio_data.unsubmitted_oe_range_end)
8905                                 __endio_write_update_ordered(inode,
8906                                         dio_data.unsubmitted_oe_range_start,
8907                                         dio_data.unsubmitted_oe_range_end -
8908                                         dio_data.unsubmitted_oe_range_start,
8909                                         false);
8910                 } else if (ret >= 0 && (size_t)ret < count)
8911                         btrfs_delalloc_release_space(inode, data_reserved,
8912                                         offset, count - (size_t)ret, true);
8913                 btrfs_delalloc_release_extents(BTRFS_I(inode), count);
8914         }
8915 out:
8916         if (wakeup)
8917                 inode_dio_end(inode);
8918         if (relock)
8919                 inode_lock(inode);
8920
8921         extent_changeset_free(data_reserved);
8922         return ret;
8923 }
8924
8925 #define BTRFS_FIEMAP_FLAGS      (FIEMAP_FLAG_SYNC)
8926
8927 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
8928                 __u64 start, __u64 len)
8929 {
8930         int     ret;
8931
8932         ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS);
8933         if (ret)
8934                 return ret;
8935
8936         return extent_fiemap(inode, fieinfo, start, len);
8937 }
8938
8939 int btrfs_readpage(struct file *file, struct page *page)
8940 {
8941         struct extent_io_tree *tree;
8942         tree = &BTRFS_I(page->mapping->host)->io_tree;
8943         return extent_read_full_page(tree, page, btrfs_get_extent, 0);
8944 }
8945
8946 static int btrfs_writepage(struct page *page, struct writeback_control *wbc)
8947 {
8948         struct inode *inode = page->mapping->host;
8949         int ret;
8950
8951         if (current->flags & PF_MEMALLOC) {
8952                 redirty_page_for_writepage(wbc, page);
8953                 unlock_page(page);
8954                 return 0;
8955         }
8956
8957         /*
8958          * If we are under memory pressure we will call this directly from the
8959          * VM, we need to make sure we have the inode referenced for the ordered
8960          * extent.  If not just return like we didn't do anything.
8961          */
8962         if (!igrab(inode)) {
8963                 redirty_page_for_writepage(wbc, page);
8964                 return AOP_WRITEPAGE_ACTIVATE;
8965         }
8966         ret = extent_write_full_page(page, wbc);
8967         btrfs_add_delayed_iput(inode);
8968         return ret;
8969 }
8970
8971 static int btrfs_writepages(struct address_space *mapping,
8972                             struct writeback_control *wbc)
8973 {
8974         return extent_writepages(mapping, wbc);
8975 }
8976
8977 static int
8978 btrfs_readpages(struct file *file, struct address_space *mapping,
8979                 struct list_head *pages, unsigned nr_pages)
8980 {
8981         return extent_readpages(mapping, pages, nr_pages);
8982 }
8983
8984 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8985 {
8986         int ret = try_release_extent_mapping(page, gfp_flags);
8987         if (ret == 1) {
8988                 ClearPagePrivate(page);
8989                 set_page_private(page, 0);
8990                 put_page(page);
8991         }
8992         return ret;
8993 }
8994
8995 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8996 {
8997         if (PageWriteback(page) || PageDirty(page))
8998                 return 0;
8999         return __btrfs_releasepage(page, gfp_flags);
9000 }
9001
9002 static void btrfs_invalidatepage(struct page *page, unsigned int offset,
9003                                  unsigned int length)
9004 {
9005         struct inode *inode = page->mapping->host;
9006         struct extent_io_tree *tree;
9007         struct btrfs_ordered_extent *ordered;
9008         struct extent_state *cached_state = NULL;
9009         u64 page_start = page_offset(page);
9010         u64 page_end = page_start + PAGE_SIZE - 1;
9011         u64 start;
9012         u64 end;
9013         int inode_evicting = inode->i_state & I_FREEING;
9014
9015         /*
9016          * we have the page locked, so new writeback can't start,
9017          * and the dirty bit won't be cleared while we are here.
9018          *
9019          * Wait for IO on this page so that we can safely clear
9020          * the PagePrivate2 bit and do ordered accounting
9021          */
9022         wait_on_page_writeback(page);
9023
9024         tree = &BTRFS_I(inode)->io_tree;
9025         if (offset) {
9026                 btrfs_releasepage(page, GFP_NOFS);
9027                 return;
9028         }
9029
9030         if (!inode_evicting)
9031                 lock_extent_bits(tree, page_start, page_end, &cached_state);
9032 again:
9033         start = page_start;
9034         ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
9035                                         page_end - start + 1);
9036         if (ordered) {
9037                 end = min(page_end, ordered->file_offset + ordered->len - 1);
9038                 /*
9039                  * IO on this page will never be started, so we need
9040                  * to account for any ordered extents now
9041                  */
9042                 if (!inode_evicting)
9043                         clear_extent_bit(tree, start, end,
9044                                          EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
9045                                          EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
9046                                          EXTENT_DEFRAG, 1, 0, &cached_state);
9047                 /*
9048                  * whoever cleared the private bit is responsible
9049                  * for the finish_ordered_io
9050                  */
9051                 if (TestClearPagePrivate2(page)) {
9052                         struct btrfs_ordered_inode_tree *tree;
9053                         u64 new_len;
9054
9055                         tree = &BTRFS_I(inode)->ordered_tree;
9056
9057                         spin_lock_irq(&tree->lock);
9058                         set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
9059                         new_len = start - ordered->file_offset;
9060                         if (new_len < ordered->truncated_len)
9061                                 ordered->truncated_len = new_len;
9062                         spin_unlock_irq(&tree->lock);
9063
9064                         if (btrfs_dec_test_ordered_pending(inode, &ordered,
9065                                                            start,
9066                                                            end - start + 1, 1))
9067                                 btrfs_finish_ordered_io(ordered);
9068                 }
9069                 btrfs_put_ordered_extent(ordered);
9070                 if (!inode_evicting) {
9071                         cached_state = NULL;
9072                         lock_extent_bits(tree, start, end,
9073                                          &cached_state);
9074                 }
9075
9076                 start = end + 1;
9077                 if (start < page_end)
9078                         goto again;
9079         }
9080
9081         /*
9082          * Qgroup reserved space handler
9083          * Page here will be either
9084          * 1) Already written to disk or ordered extent already submitted
9085          *    Then its QGROUP_RESERVED bit in io_tree is already cleaned.
9086          *    Qgroup will be handled by its qgroup_record then.
9087          *    btrfs_qgroup_free_data() call will do nothing here.
9088          *
9089          * 2) Not written to disk yet
9090          *    Then btrfs_qgroup_free_data() call will clear the QGROUP_RESERVED
9091          *    bit of its io_tree, and free the qgroup reserved data space.
9092          *    Since the IO will never happen for this page.
9093          */
9094         btrfs_qgroup_free_data(inode, NULL, page_start, PAGE_SIZE);
9095         if (!inode_evicting) {
9096                 clear_extent_bit(tree, page_start, page_end, EXTENT_LOCKED |
9097                                  EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
9098                                  EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1,
9099                                  &cached_state);
9100
9101                 __btrfs_releasepage(page, GFP_NOFS);
9102         }
9103
9104         ClearPageChecked(page);
9105         if (PagePrivate(page)) {
9106                 ClearPagePrivate(page);
9107                 set_page_private(page, 0);
9108                 put_page(page);
9109         }
9110 }
9111
9112 /*
9113  * btrfs_page_mkwrite() is not allowed to change the file size as it gets
9114  * called from a page fault handler when a page is first dirtied. Hence we must
9115  * be careful to check for EOF conditions here. We set the page up correctly
9116  * for a written page which means we get ENOSPC checking when writing into
9117  * holes and correct delalloc and unwritten extent mapping on filesystems that
9118  * support these features.
9119  *
9120  * We are not allowed to take the i_mutex here so we have to play games to
9121  * protect against truncate races as the page could now be beyond EOF.  Because
9122  * truncate_setsize() writes the inode size before removing pages, once we have
9123  * the page lock we can determine safely if the page is beyond EOF. If it is not
9124  * beyond EOF, then the page is guaranteed safe against truncation until we
9125  * unlock the page.
9126  */
9127 vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf)
9128 {
9129         struct page *page = vmf->page;
9130         struct inode *inode = file_inode(vmf->vma->vm_file);
9131         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9132         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
9133         struct btrfs_ordered_extent *ordered;
9134         struct extent_state *cached_state = NULL;
9135         struct extent_changeset *data_reserved = NULL;
9136         char *kaddr;
9137         unsigned long zero_start;
9138         loff_t size;
9139         vm_fault_t ret;
9140         int ret2;
9141         int reserved = 0;
9142         u64 reserved_space;
9143         u64 page_start;
9144         u64 page_end;
9145         u64 end;
9146
9147         reserved_space = PAGE_SIZE;
9148
9149         sb_start_pagefault(inode->i_sb);
9150         page_start = page_offset(page);
9151         page_end = page_start + PAGE_SIZE - 1;
9152         end = page_end;
9153
9154         /*
9155          * Reserving delalloc space after obtaining the page lock can lead to
9156          * deadlock. For example, if a dirty page is locked by this function
9157          * and the call to btrfs_delalloc_reserve_space() ends up triggering
9158          * dirty page write out, then the btrfs_writepage() function could
9159          * end up waiting indefinitely to get a lock on the page currently
9160          * being processed by btrfs_page_mkwrite() function.
9161          */
9162         ret2 = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
9163                                            reserved_space);
9164         if (!ret2) {
9165                 ret2 = file_update_time(vmf->vma->vm_file);
9166                 reserved = 1;
9167         }
9168         if (ret2) {
9169                 ret = vmf_error(ret2);
9170                 if (reserved)
9171                         goto out;
9172                 goto out_noreserve;
9173         }
9174
9175         ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
9176 again:
9177         lock_page(page);
9178         size = i_size_read(inode);
9179
9180         if ((page->mapping != inode->i_mapping) ||
9181             (page_start >= size)) {
9182                 /* page got truncated out from underneath us */
9183                 goto out_unlock;
9184         }
9185         wait_on_page_writeback(page);
9186
9187         lock_extent_bits(io_tree, page_start, page_end, &cached_state);
9188         set_page_extent_mapped(page);
9189
9190         /*
9191          * we can't set the delalloc bits if there are pending ordered
9192          * extents.  Drop our locks and wait for them to finish
9193          */
9194         ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
9195                         PAGE_SIZE);
9196         if (ordered) {
9197                 unlock_extent_cached(io_tree, page_start, page_end,
9198                                      &cached_state);
9199                 unlock_page(page);
9200                 btrfs_start_ordered_extent(inode, ordered, 1);
9201                 btrfs_put_ordered_extent(ordered);
9202                 goto again;
9203         }
9204
9205         if (page->index == ((size - 1) >> PAGE_SHIFT)) {
9206                 reserved_space = round_up(size - page_start,
9207                                           fs_info->sectorsize);
9208                 if (reserved_space < PAGE_SIZE) {
9209                         end = page_start + reserved_space - 1;
9210                         btrfs_delalloc_release_space(inode, data_reserved,
9211                                         page_start, PAGE_SIZE - reserved_space,
9212                                         true);
9213                 }
9214         }
9215
9216         /*
9217          * page_mkwrite gets called when the page is firstly dirtied after it's
9218          * faulted in, but write(2) could also dirty a page and set delalloc
9219          * bits, thus in this case for space account reason, we still need to
9220          * clear any delalloc bits within this page range since we have to
9221          * reserve data&meta space before lock_page() (see above comments).
9222          */
9223         clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end,
9224                           EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
9225                           EXTENT_DEFRAG, 0, 0, &cached_state);
9226
9227         ret2 = btrfs_set_extent_delalloc(inode, page_start, end, 0,
9228                                         &cached_state);
9229         if (ret2) {
9230                 unlock_extent_cached(io_tree, page_start, page_end,
9231                                      &cached_state);
9232                 ret = VM_FAULT_SIGBUS;
9233                 goto out_unlock;
9234         }
9235         ret2 = 0;
9236
9237         /* page is wholly or partially inside EOF */
9238         if (page_start + PAGE_SIZE > size)
9239                 zero_start = offset_in_page(size);
9240         else
9241                 zero_start = PAGE_SIZE;
9242
9243         if (zero_start != PAGE_SIZE) {
9244                 kaddr = kmap(page);
9245                 memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start);
9246                 flush_dcache_page(page);
9247                 kunmap(page);
9248         }
9249         ClearPageChecked(page);
9250         set_page_dirty(page);
9251         SetPageUptodate(page);
9252
9253         btrfs_set_inode_last_sub_trans(BTRFS_I(inode));
9254
9255         unlock_extent_cached(io_tree, page_start, page_end, &cached_state);
9256
9257         if (!ret2) {
9258                 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE);
9259                 sb_end_pagefault(inode->i_sb);
9260                 extent_changeset_free(data_reserved);
9261                 return VM_FAULT_LOCKED;
9262         }
9263
9264 out_unlock:
9265         unlock_page(page);
9266 out:
9267         btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE);
9268         btrfs_delalloc_release_space(inode, data_reserved, page_start,
9269                                      reserved_space, (ret != 0));
9270 out_noreserve:
9271         sb_end_pagefault(inode->i_sb);
9272         extent_changeset_free(data_reserved);
9273         return ret;
9274 }
9275
9276 static int btrfs_truncate(struct inode *inode, bool skip_writeback)
9277 {
9278         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9279         struct btrfs_root *root = BTRFS_I(inode)->root;
9280         struct btrfs_block_rsv *rsv;
9281         int ret;
9282         struct btrfs_trans_handle *trans;
9283         u64 mask = fs_info->sectorsize - 1;
9284         u64 min_size = btrfs_calc_metadata_size(fs_info, 1);
9285
9286         if (!skip_writeback) {
9287                 ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask),
9288                                                (u64)-1);
9289                 if (ret)
9290                         return ret;
9291         }
9292
9293         /*
9294          * Yes ladies and gentlemen, this is indeed ugly.  We have a couple of
9295          * things going on here:
9296          *
9297          * 1) We need to reserve space to update our inode.
9298          *
9299          * 2) We need to have something to cache all the space that is going to
9300          * be free'd up by the truncate operation, but also have some slack
9301          * space reserved in case it uses space during the truncate (thank you
9302          * very much snapshotting).
9303          *
9304          * And we need these to be separate.  The fact is we can use a lot of
9305          * space doing the truncate, and we have no earthly idea how much space
9306          * we will use, so we need the truncate reservation to be separate so it
9307          * doesn't end up using space reserved for updating the inode.  We also
9308          * need to be able to stop the transaction and start a new one, which
9309          * means we need to be able to update the inode several times, and we
9310          * have no idea of knowing how many times that will be, so we can't just
9311          * reserve 1 item for the entirety of the operation, so that has to be
9312          * done separately as well.
9313          *
9314          * So that leaves us with
9315          *
9316          * 1) rsv - for the truncate reservation, which we will steal from the
9317          * transaction reservation.
9318          * 2) fs_info->trans_block_rsv - this will have 1 items worth left for
9319          * updating the inode.
9320          */
9321         rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
9322         if (!rsv)
9323                 return -ENOMEM;
9324         rsv->size = min_size;
9325         rsv->failfast = 1;
9326
9327         /*
9328          * 1 for the truncate slack space
9329          * 1 for updating the inode.
9330          */
9331         trans = btrfs_start_transaction(root, 2);
9332         if (IS_ERR(trans)) {
9333                 ret = PTR_ERR(trans);
9334                 goto out;
9335         }
9336
9337         /* Migrate the slack space for the truncate to our reserve */
9338         ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
9339                                       min_size, false);
9340         BUG_ON(ret);
9341
9342         /*
9343          * So if we truncate and then write and fsync we normally would just
9344          * write the extents that changed, which is a problem if we need to
9345          * first truncate that entire inode.  So set this flag so we write out
9346          * all of the extents in the inode to the sync log so we're completely
9347          * safe.
9348          */
9349         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
9350         trans->block_rsv = rsv;
9351
9352         while (1) {
9353                 ret = btrfs_truncate_inode_items(trans, root, inode,
9354                                                  inode->i_size,
9355                                                  BTRFS_EXTENT_DATA_KEY);
9356                 trans->block_rsv = &fs_info->trans_block_rsv;
9357                 if (ret != -ENOSPC && ret != -EAGAIN)
9358                         break;
9359
9360                 ret = btrfs_update_inode(trans, root, inode);
9361                 if (ret)
9362                         break;
9363
9364                 btrfs_end_transaction(trans);
9365                 btrfs_btree_balance_dirty(fs_info);
9366
9367                 trans = btrfs_start_transaction(root, 2);
9368                 if (IS_ERR(trans)) {
9369                         ret = PTR_ERR(trans);
9370                         trans = NULL;
9371                         break;
9372                 }
9373
9374                 btrfs_block_rsv_release(fs_info, rsv, -1);
9375                 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
9376                                               rsv, min_size, false);
9377                 BUG_ON(ret);    /* shouldn't happen */
9378                 trans->block_rsv = rsv;
9379         }
9380
9381         /*
9382          * We can't call btrfs_truncate_block inside a trans handle as we could
9383          * deadlock with freeze, if we got NEED_TRUNCATE_BLOCK then we know
9384          * we've truncated everything except the last little bit, and can do
9385          * btrfs_truncate_block and then update the disk_i_size.
9386          */
9387         if (ret == NEED_TRUNCATE_BLOCK) {
9388                 btrfs_end_transaction(trans);
9389                 btrfs_btree_balance_dirty(fs_info);
9390
9391                 ret = btrfs_truncate_block(inode, inode->i_size, 0, 0);
9392                 if (ret)
9393                         goto out;
9394                 trans = btrfs_start_transaction(root, 1);
9395                 if (IS_ERR(trans)) {
9396                         ret = PTR_ERR(trans);
9397                         goto out;
9398                 }
9399                 btrfs_ordered_update_i_size(inode, inode->i_size, NULL);
9400         }
9401
9402         if (trans) {
9403                 int ret2;
9404
9405                 trans->block_rsv = &fs_info->trans_block_rsv;
9406                 ret2 = btrfs_update_inode(trans, root, inode);
9407                 if (ret2 && !ret)
9408                         ret = ret2;
9409
9410                 ret2 = btrfs_end_transaction(trans);
9411                 if (ret2 && !ret)
9412                         ret = ret2;
9413                 btrfs_btree_balance_dirty(fs_info);
9414         }
9415 out:
9416         btrfs_free_block_rsv(fs_info, rsv);
9417
9418         return ret;
9419 }
9420
9421 /*
9422  * create a new subvolume directory/inode (helper for the ioctl).
9423  */
9424 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans,
9425                              struct btrfs_root *new_root,
9426                              struct btrfs_root *parent_root,
9427                              u64 new_dirid)
9428 {
9429         struct inode *inode;
9430         int err;
9431         u64 index = 0;
9432
9433         inode = btrfs_new_inode(trans, new_root, NULL, "..", 2,
9434                                 new_dirid, new_dirid,
9435                                 S_IFDIR | (~current_umask() & S_IRWXUGO),
9436                                 &index);
9437         if (IS_ERR(inode))
9438                 return PTR_ERR(inode);
9439         inode->i_op = &btrfs_dir_inode_operations;
9440         inode->i_fop = &btrfs_dir_file_operations;
9441
9442         set_nlink(inode, 1);
9443         btrfs_i_size_write(BTRFS_I(inode), 0);
9444         unlock_new_inode(inode);
9445
9446         err = btrfs_subvol_inherit_props(trans, new_root, parent_root);
9447         if (err)
9448                 btrfs_err(new_root->fs_info,
9449                           "error inheriting subvolume %llu properties: %d",
9450                           new_root->root_key.objectid, err);
9451
9452         err = btrfs_update_inode(trans, new_root, inode);
9453
9454         iput(inode);
9455         return err;
9456 }
9457
9458 struct inode *btrfs_alloc_inode(struct super_block *sb)
9459 {
9460         struct btrfs_fs_info *fs_info = btrfs_sb(sb);
9461         struct btrfs_inode *ei;
9462         struct inode *inode;
9463
9464         ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_KERNEL);
9465         if (!ei)
9466                 return NULL;
9467
9468         ei->root = NULL;
9469         ei->generation = 0;
9470         ei->last_trans = 0;
9471         ei->last_sub_trans = 0;
9472         ei->logged_trans = 0;
9473         ei->delalloc_bytes = 0;
9474         ei->new_delalloc_bytes = 0;
9475         ei->defrag_bytes = 0;
9476         ei->disk_i_size = 0;
9477         ei->flags = 0;
9478         ei->csum_bytes = 0;
9479         ei->index_cnt = (u64)-1;
9480         ei->dir_index = 0;
9481         ei->last_unlink_trans = 0;
9482         ei->last_log_commit = 0;
9483
9484         spin_lock_init(&ei->lock);
9485         ei->outstanding_extents = 0;
9486         if (sb->s_magic != BTRFS_TEST_MAGIC)
9487                 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv,
9488                                               BTRFS_BLOCK_RSV_DELALLOC);
9489         ei->runtime_flags = 0;
9490         ei->prop_compress = BTRFS_COMPRESS_NONE;
9491         ei->defrag_compress = BTRFS_COMPRESS_NONE;
9492
9493         ei->delayed_node = NULL;
9494
9495         ei->i_otime.tv_sec = 0;
9496         ei->i_otime.tv_nsec = 0;
9497
9498         inode = &ei->vfs_inode;
9499         extent_map_tree_init(&ei->extent_tree);
9500         extent_io_tree_init(fs_info, &ei->io_tree, IO_TREE_INODE_IO, inode);
9501         extent_io_tree_init(fs_info, &ei->io_failure_tree,
9502                             IO_TREE_INODE_IO_FAILURE, inode);
9503         ei->io_tree.track_uptodate = true;
9504         ei->io_failure_tree.track_uptodate = true;
9505         atomic_set(&ei->sync_writers, 0);
9506         mutex_init(&ei->log_mutex);
9507         mutex_init(&ei->delalloc_mutex);
9508         btrfs_ordered_inode_tree_init(&ei->ordered_tree);
9509         INIT_LIST_HEAD(&ei->delalloc_inodes);
9510         INIT_LIST_HEAD(&ei->delayed_iput);
9511         RB_CLEAR_NODE(&ei->rb_node);
9512         init_rwsem(&ei->dio_sem);
9513
9514         return inode;
9515 }
9516
9517 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
9518 void btrfs_test_destroy_inode(struct inode *inode)
9519 {
9520         btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
9521         kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
9522 }
9523 #endif
9524
9525 void btrfs_free_inode(struct inode *inode)
9526 {
9527         kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
9528 }
9529
9530 void btrfs_destroy_inode(struct inode *inode)
9531 {
9532         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9533         struct btrfs_ordered_extent *ordered;
9534         struct btrfs_root *root = BTRFS_I(inode)->root;
9535
9536         WARN_ON(!hlist_empty(&inode->i_dentry));
9537         WARN_ON(inode->i_data.nrpages);
9538         WARN_ON(BTRFS_I(inode)->block_rsv.reserved);
9539         WARN_ON(BTRFS_I(inode)->block_rsv.size);
9540         WARN_ON(BTRFS_I(inode)->outstanding_extents);
9541         WARN_ON(BTRFS_I(inode)->delalloc_bytes);
9542         WARN_ON(BTRFS_I(inode)->new_delalloc_bytes);
9543         WARN_ON(BTRFS_I(inode)->csum_bytes);
9544         WARN_ON(BTRFS_I(inode)->defrag_bytes);
9545
9546         /*
9547          * This can happen where we create an inode, but somebody else also
9548          * created the same inode and we need to destroy the one we already
9549          * created.
9550          */
9551         if (!root)
9552                 return;
9553
9554         while (1) {
9555                 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
9556                 if (!ordered)
9557                         break;
9558                 else {
9559                         btrfs_err(fs_info,
9560                                   "found ordered extent %llu %llu on inode cleanup",
9561                                   ordered->file_offset, ordered->len);
9562                         btrfs_remove_ordered_extent(inode, ordered);
9563                         btrfs_put_ordered_extent(ordered);
9564                         btrfs_put_ordered_extent(ordered);
9565                 }
9566         }
9567         btrfs_qgroup_check_reserved_leak(BTRFS_I(inode));
9568         inode_tree_del(inode);
9569         btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
9570 }
9571
9572 int btrfs_drop_inode(struct inode *inode)
9573 {
9574         struct btrfs_root *root = BTRFS_I(inode)->root;
9575
9576         if (root == NULL)
9577                 return 1;
9578
9579         /* the snap/subvol tree is on deleting */
9580         if (btrfs_root_refs(&root->root_item) == 0)
9581                 return 1;
9582         else
9583                 return generic_drop_inode(inode);
9584 }
9585
9586 static void init_once(void *foo)
9587 {
9588         struct btrfs_inode *ei = (struct btrfs_inode *) foo;
9589
9590         inode_init_once(&ei->vfs_inode);
9591 }
9592
9593 void __cold btrfs_destroy_cachep(void)
9594 {
9595         /*
9596          * Make sure all delayed rcu free inodes are flushed before we
9597          * destroy cache.
9598          */
9599         rcu_barrier();
9600         kmem_cache_destroy(btrfs_inode_cachep);
9601         kmem_cache_destroy(btrfs_trans_handle_cachep);
9602         kmem_cache_destroy(btrfs_path_cachep);
9603         kmem_cache_destroy(btrfs_free_space_cachep);
9604         kmem_cache_destroy(btrfs_free_space_bitmap_cachep);
9605 }
9606
9607 int __init btrfs_init_cachep(void)
9608 {
9609         btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
9610                         sizeof(struct btrfs_inode), 0,
9611                         SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT,
9612                         init_once);
9613         if (!btrfs_inode_cachep)
9614                 goto fail;
9615
9616         btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle",
9617                         sizeof(struct btrfs_trans_handle), 0,
9618                         SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL);
9619         if (!btrfs_trans_handle_cachep)
9620                 goto fail;
9621
9622         btrfs_path_cachep = kmem_cache_create("btrfs_path",
9623                         sizeof(struct btrfs_path), 0,
9624                         SLAB_MEM_SPREAD, NULL);
9625         if (!btrfs_path_cachep)
9626                 goto fail;
9627
9628         btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space",
9629                         sizeof(struct btrfs_free_space), 0,
9630                         SLAB_MEM_SPREAD, NULL);
9631         if (!btrfs_free_space_cachep)
9632                 goto fail;
9633
9634         btrfs_free_space_bitmap_cachep = kmem_cache_create("btrfs_free_space_bitmap",
9635                                                         PAGE_SIZE, PAGE_SIZE,
9636                                                         SLAB_MEM_SPREAD, NULL);
9637         if (!btrfs_free_space_bitmap_cachep)
9638                 goto fail;
9639
9640         return 0;
9641 fail:
9642         btrfs_destroy_cachep();
9643         return -ENOMEM;
9644 }
9645
9646 static int btrfs_getattr(const struct path *path, struct kstat *stat,
9647                          u32 request_mask, unsigned int flags)
9648 {
9649         u64 delalloc_bytes;
9650         struct inode *inode = d_inode(path->dentry);
9651         u32 blocksize = inode->i_sb->s_blocksize;
9652         u32 bi_flags = BTRFS_I(inode)->flags;
9653
9654         stat->result_mask |= STATX_BTIME;
9655         stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec;
9656         stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec;
9657         if (bi_flags & BTRFS_INODE_APPEND)
9658                 stat->attributes |= STATX_ATTR_APPEND;
9659         if (bi_flags & BTRFS_INODE_COMPRESS)
9660                 stat->attributes |= STATX_ATTR_COMPRESSED;
9661         if (bi_flags & BTRFS_INODE_IMMUTABLE)
9662                 stat->attributes |= STATX_ATTR_IMMUTABLE;
9663         if (bi_flags & BTRFS_INODE_NODUMP)
9664                 stat->attributes |= STATX_ATTR_NODUMP;
9665
9666         stat->attributes_mask |= (STATX_ATTR_APPEND |
9667                                   STATX_ATTR_COMPRESSED |
9668                                   STATX_ATTR_IMMUTABLE |
9669                                   STATX_ATTR_NODUMP);
9670
9671         generic_fillattr(inode, stat);
9672         stat->dev = BTRFS_I(inode)->root->anon_dev;
9673
9674         spin_lock(&BTRFS_I(inode)->lock);
9675         delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes;
9676         spin_unlock(&BTRFS_I(inode)->lock);
9677         stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) +
9678                         ALIGN(delalloc_bytes, blocksize)) >> 9;
9679         return 0;
9680 }
9681
9682 static int btrfs_rename_exchange(struct inode *old_dir,
9683                               struct dentry *old_dentry,
9684                               struct inode *new_dir,
9685                               struct dentry *new_dentry)
9686 {
9687         struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
9688         struct btrfs_trans_handle *trans;
9689         struct btrfs_root *root = BTRFS_I(old_dir)->root;
9690         struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9691         struct inode *new_inode = new_dentry->d_inode;
9692         struct inode *old_inode = old_dentry->d_inode;
9693         struct timespec64 ctime = current_time(old_inode);
9694         u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
9695         u64 new_ino = btrfs_ino(BTRFS_I(new_inode));
9696         u64 old_idx = 0;
9697         u64 new_idx = 0;
9698         int ret;
9699         int ret2;
9700         bool root_log_pinned = false;
9701         bool dest_log_pinned = false;
9702
9703         /*
9704          * For non-subvolumes allow exchange only within one subvolume, in the
9705          * same inode namespace. Two subvolumes (represented as directory) can
9706          * be exchanged as they're a logical link and have a fixed inode number.
9707          */
9708         if (root != dest &&
9709             (old_ino != BTRFS_FIRST_FREE_OBJECTID ||
9710              new_ino != BTRFS_FIRST_FREE_OBJECTID))
9711                 return -EXDEV;
9712
9713         /* close the race window with snapshot create/destroy ioctl */
9714         if (old_ino == BTRFS_FIRST_FREE_OBJECTID ||
9715             new_ino == BTRFS_FIRST_FREE_OBJECTID)
9716                 down_read(&fs_info->subvol_sem);
9717
9718         /*
9719          * We want to reserve the absolute worst case amount of items.  So if
9720          * both inodes are subvols and we need to unlink them then that would
9721          * require 4 item modifications, but if they are both normal inodes it
9722          * would require 5 item modifications, so we'll assume their normal
9723          * inodes.  So 5 * 2 is 10, plus 2 for the new links, so 12 total items
9724          * should cover the worst case number of items we'll modify.
9725          */
9726         trans = btrfs_start_transaction(root, 12);
9727         if (IS_ERR(trans)) {
9728                 ret = PTR_ERR(trans);
9729                 goto out_notrans;
9730         }
9731
9732         if (dest != root)
9733                 btrfs_record_root_in_trans(trans, dest);
9734
9735         /*
9736          * We need to find a free sequence number both in the source and
9737          * in the destination directory for the exchange.
9738          */
9739         ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx);
9740         if (ret)
9741                 goto out_fail;
9742         ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx);
9743         if (ret)
9744                 goto out_fail;
9745
9746         BTRFS_I(old_inode)->dir_index = 0ULL;
9747         BTRFS_I(new_inode)->dir_index = 0ULL;
9748
9749         /* Reference for the source. */
9750         if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
9751                 /* force full log commit if subvolume involved. */
9752                 btrfs_set_log_full_commit(trans);
9753         } else {
9754                 ret = btrfs_insert_inode_ref(trans, dest,
9755                                              new_dentry->d_name.name,
9756                                              new_dentry->d_name.len,
9757                                              old_ino,
9758                                              btrfs_ino(BTRFS_I(new_dir)),
9759                                              old_idx);
9760                 if (ret)
9761                         goto out_fail;
9762         }
9763
9764         /* And now for the dest. */
9765         if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
9766                 /* force full log commit if subvolume involved. */
9767                 btrfs_set_log_full_commit(trans);
9768         } else {
9769                 ret = btrfs_insert_inode_ref(trans, root,
9770                                              old_dentry->d_name.name,
9771                                              old_dentry->d_name.len,
9772                                              new_ino,
9773                                              btrfs_ino(BTRFS_I(old_dir)),
9774                                              new_idx);
9775                 if (ret)
9776                         goto out_fail;
9777         }
9778
9779         /* Update inode version and ctime/mtime. */
9780         inode_inc_iversion(old_dir);
9781         inode_inc_iversion(new_dir);
9782         inode_inc_iversion(old_inode);
9783         inode_inc_iversion(new_inode);
9784         old_dir->i_ctime = old_dir->i_mtime = ctime;
9785         new_dir->i_ctime = new_dir->i_mtime = ctime;
9786         old_inode->i_ctime = ctime;
9787         new_inode->i_ctime = ctime;
9788
9789         if (old_dentry->d_parent != new_dentry->d_parent) {
9790                 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
9791                                 BTRFS_I(old_inode), 1);
9792                 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir),
9793                                 BTRFS_I(new_inode), 1);
9794         }
9795
9796         /*
9797          * Now pin the logs of the roots. We do it to ensure that no other task
9798          * can sync the logs while we are in progress with the rename, because
9799          * that could result in an inconsistency in case any of the inodes that
9800          * are part of this rename operation were logged before.
9801          *
9802          * We pin the logs even if at this precise moment none of the inodes was
9803          * logged before. This is because right after we checked for that, some
9804          * other task fsyncing some other inode not involved with this rename
9805          * operation could log that one of our inodes exists.
9806          *
9807          * We don't need to pin the logs before the above calls to
9808          * btrfs_insert_inode_ref(), since those don't ever need to change a log.
9809          */
9810         if (old_ino != BTRFS_FIRST_FREE_OBJECTID) {
9811                 btrfs_pin_log_trans(root);
9812                 root_log_pinned = true;
9813         }
9814         if (new_ino != BTRFS_FIRST_FREE_OBJECTID) {
9815                 btrfs_pin_log_trans(dest);
9816                 dest_log_pinned = true;
9817         }
9818
9819         /* src is a subvolume */
9820         if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
9821                 ret = btrfs_unlink_subvol(trans, old_dir, old_dentry);
9822         } else { /* src is an inode */
9823                 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
9824                                            BTRFS_I(old_dentry->d_inode),
9825                                            old_dentry->d_name.name,
9826                                            old_dentry->d_name.len);
9827                 if (!ret)
9828                         ret = btrfs_update_inode(trans, root, old_inode);
9829         }
9830         if (ret) {
9831                 btrfs_abort_transaction(trans, ret);
9832                 goto out_fail;
9833         }
9834
9835         /* dest is a subvolume */
9836         if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
9837                 ret = btrfs_unlink_subvol(trans, new_dir, new_dentry);
9838         } else { /* dest is an inode */
9839                 ret = __btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
9840                                            BTRFS_I(new_dentry->d_inode),
9841                                            new_dentry->d_name.name,
9842                                            new_dentry->d_name.len);
9843                 if (!ret)
9844                         ret = btrfs_update_inode(trans, dest, new_inode);
9845         }
9846         if (ret) {
9847                 btrfs_abort_transaction(trans, ret);
9848                 goto out_fail;
9849         }
9850
9851         ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
9852                              new_dentry->d_name.name,
9853                              new_dentry->d_name.len, 0, old_idx);
9854         if (ret) {
9855                 btrfs_abort_transaction(trans, ret);
9856                 goto out_fail;
9857         }
9858
9859         ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
9860                              old_dentry->d_name.name,
9861                              old_dentry->d_name.len, 0, new_idx);
9862         if (ret) {
9863                 btrfs_abort_transaction(trans, ret);
9864                 goto out_fail;
9865         }
9866
9867         if (old_inode->i_nlink == 1)
9868                 BTRFS_I(old_inode)->dir_index = old_idx;
9869         if (new_inode->i_nlink == 1)
9870                 BTRFS_I(new_inode)->dir_index = new_idx;
9871
9872         if (root_log_pinned) {
9873                 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
9874                                    new_dentry->d_parent);
9875                 btrfs_end_log_trans(root);
9876                 root_log_pinned = false;
9877         }
9878         if (dest_log_pinned) {
9879                 btrfs_log_new_name(trans, BTRFS_I(new_inode), BTRFS_I(new_dir),
9880                                    old_dentry->d_parent);
9881                 btrfs_end_log_trans(dest);
9882                 dest_log_pinned = false;
9883         }
9884 out_fail:
9885         /*
9886          * If we have pinned a log and an error happened, we unpin tasks
9887          * trying to sync the log and force them to fallback to a transaction
9888          * commit if the log currently contains any of the inodes involved in
9889          * this rename operation (to ensure we do not persist a log with an
9890          * inconsistent state for any of these inodes or leading to any
9891          * inconsistencies when replayed). If the transaction was aborted, the
9892          * abortion reason is propagated to userspace when attempting to commit
9893          * the transaction. If the log does not contain any of these inodes, we
9894          * allow the tasks to sync it.
9895          */
9896         if (ret && (root_log_pinned || dest_log_pinned)) {
9897                 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
9898                     btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
9899                     btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
9900                     (new_inode &&
9901                      btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
9902                         btrfs_set_log_full_commit(trans);
9903
9904                 if (root_log_pinned) {
9905                         btrfs_end_log_trans(root);
9906                         root_log_pinned = false;
9907                 }
9908                 if (dest_log_pinned) {
9909                         btrfs_end_log_trans(dest);
9910                         dest_log_pinned = false;
9911                 }
9912         }
9913         ret2 = btrfs_end_transaction(trans);
9914         ret = ret ? ret : ret2;
9915 out_notrans:
9916         if (new_ino == BTRFS_FIRST_FREE_OBJECTID ||
9917             old_ino == BTRFS_FIRST_FREE_OBJECTID)
9918                 up_read(&fs_info->subvol_sem);
9919
9920         return ret;
9921 }
9922
9923 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans,
9924                                      struct btrfs_root *root,
9925                                      struct inode *dir,
9926                                      struct dentry *dentry)
9927 {
9928         int ret;
9929         struct inode *inode;
9930         u64 objectid;
9931         u64 index;
9932
9933         ret = btrfs_find_free_objectid(root, &objectid);
9934         if (ret)
9935                 return ret;
9936
9937         inode = btrfs_new_inode(trans, root, dir,
9938                                 dentry->d_name.name,
9939                                 dentry->d_name.len,
9940                                 btrfs_ino(BTRFS_I(dir)),
9941                                 objectid,
9942                                 S_IFCHR | WHITEOUT_MODE,
9943                                 &index);
9944
9945         if (IS_ERR(inode)) {
9946                 ret = PTR_ERR(inode);
9947                 return ret;
9948         }
9949
9950         inode->i_op = &btrfs_special_inode_operations;
9951         init_special_inode(inode, inode->i_mode,
9952                 WHITEOUT_DEV);
9953
9954         ret = btrfs_init_inode_security(trans, inode, dir,
9955                                 &dentry->d_name);
9956         if (ret)
9957                 goto out;
9958
9959         ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
9960                                 BTRFS_I(inode), 0, index);
9961         if (ret)
9962                 goto out;
9963
9964         ret = btrfs_update_inode(trans, root, inode);
9965 out:
9966         unlock_new_inode(inode);
9967         if (ret)
9968                 inode_dec_link_count(inode);
9969         iput(inode);
9970
9971         return ret;
9972 }
9973
9974 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry,
9975                            struct inode *new_dir, struct dentry *new_dentry,
9976                            unsigned int flags)
9977 {
9978         struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
9979         struct btrfs_trans_handle *trans;
9980         unsigned int trans_num_items;
9981         struct btrfs_root *root = BTRFS_I(old_dir)->root;
9982         struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9983         struct inode *new_inode = d_inode(new_dentry);
9984         struct inode *old_inode = d_inode(old_dentry);
9985         u64 index = 0;
9986         int ret;
9987         int ret2;
9988         u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
9989         bool log_pinned = false;
9990
9991         if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
9992                 return -EPERM;
9993
9994         /* we only allow rename subvolume link between subvolumes */
9995         if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
9996                 return -EXDEV;
9997
9998         if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||
9999             (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID))
10000                 return -ENOTEMPTY;
10001
10002         if (S_ISDIR(old_inode->i_mode) && new_inode &&
10003             new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
10004                 return -ENOTEMPTY;
10005
10006
10007         /* check for collisions, even if the  name isn't there */
10008         ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino,
10009                              new_dentry->d_name.name,
10010                              new_dentry->d_name.len);
10011
10012         if (ret) {
10013                 if (ret == -EEXIST) {
10014                         /* we shouldn't get
10015                          * eexist without a new_inode */
10016                         if (WARN_ON(!new_inode)) {
10017                                 return ret;
10018                         }
10019                 } else {
10020                         /* maybe -EOVERFLOW */
10021                         return ret;
10022                 }
10023         }
10024         ret = 0;
10025
10026         /*
10027          * we're using rename to replace one file with another.  Start IO on it
10028          * now so  we don't add too much work to the end of the transaction
10029          */
10030         if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
10031                 filemap_flush(old_inode->i_mapping);
10032
10033         /* close the racy window with snapshot create/destroy ioctl */
10034         if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
10035                 down_read(&fs_info->subvol_sem);
10036         /*
10037          * We want to reserve the absolute worst case amount of items.  So if
10038          * both inodes are subvols and we need to unlink them then that would
10039          * require 4 item modifications, but if they are both normal inodes it
10040          * would require 5 item modifications, so we'll assume they are normal
10041          * inodes.  So 5 * 2 is 10, plus 1 for the new link, so 11 total items
10042          * should cover the worst case number of items we'll modify.
10043          * If our rename has the whiteout flag, we need more 5 units for the
10044          * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item
10045          * when selinux is enabled).
10046          */
10047         trans_num_items = 11;
10048         if (flags & RENAME_WHITEOUT)
10049                 trans_num_items += 5;
10050         trans = btrfs_start_transaction(root, trans_num_items);
10051         if (IS_ERR(trans)) {
10052                 ret = PTR_ERR(trans);
10053                 goto out_notrans;
10054         }
10055
10056         if (dest != root)
10057                 btrfs_record_root_in_trans(trans, dest);
10058
10059         ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index);
10060         if (ret)
10061                 goto out_fail;
10062
10063         BTRFS_I(old_inode)->dir_index = 0ULL;
10064         if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
10065                 /* force full log commit if subvolume involved. */
10066                 btrfs_set_log_full_commit(trans);
10067         } else {
10068                 ret = btrfs_insert_inode_ref(trans, dest,
10069                                              new_dentry->d_name.name,
10070                                              new_dentry->d_name.len,
10071                                              old_ino,
10072                                              btrfs_ino(BTRFS_I(new_dir)), index);
10073                 if (ret)
10074                         goto out_fail;
10075         }
10076
10077         inode_inc_iversion(old_dir);
10078         inode_inc_iversion(new_dir);
10079         inode_inc_iversion(old_inode);
10080         old_dir->i_ctime = old_dir->i_mtime =
10081         new_dir->i_ctime = new_dir->i_mtime =
10082         old_inode->i_ctime = current_time(old_dir);
10083
10084         if (old_dentry->d_parent != new_dentry->d_parent)
10085                 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
10086                                 BTRFS_I(old_inode), 1);
10087
10088         if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
10089                 ret = btrfs_unlink_subvol(trans, old_dir, old_dentry);
10090         } else {
10091                 /*
10092                  * Now pin the log. We do it to ensure that no other task can
10093                  * sync the log while we are in progress with the rename, as
10094                  * that could result in an inconsistency in case any of the
10095                  * inodes that are part of this rename operation were logged
10096                  * before.
10097                  *
10098                  * We pin the log even if at this precise moment none of the
10099                  * inodes was logged before. This is because right after we
10100                  * checked for that, some other task fsyncing some other inode
10101                  * not involved with this rename operation could log that one of
10102                  * our inodes exists.
10103                  *
10104                  * We don't need to pin the logs before the above call to
10105                  * btrfs_insert_inode_ref(), since that does not need to change
10106                  * a log.
10107                  */
10108                 btrfs_pin_log_trans(root);
10109                 log_pinned = true;
10110                 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
10111                                         BTRFS_I(d_inode(old_dentry)),
10112                                         old_dentry->d_name.name,
10113                                         old_dentry->d_name.len);
10114                 if (!ret)
10115                         ret = btrfs_update_inode(trans, root, old_inode);
10116         }
10117         if (ret) {
10118                 btrfs_abort_transaction(trans, ret);
10119                 goto out_fail;
10120         }
10121
10122         if (new_inode) {
10123                 inode_inc_iversion(new_inode);
10124                 new_inode->i_ctime = current_time(new_inode);
10125                 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) ==
10126                              BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
10127                         ret = btrfs_unlink_subvol(trans, new_dir, new_dentry);
10128                         BUG_ON(new_inode->i_nlink == 0);
10129                 } else {
10130                         ret = btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
10131                                                  BTRFS_I(d_inode(new_dentry)),
10132                                                  new_dentry->d_name.name,
10133                                                  new_dentry->d_name.len);
10134                 }
10135                 if (!ret && new_inode->i_nlink == 0)
10136                         ret = btrfs_orphan_add(trans,
10137                                         BTRFS_I(d_inode(new_dentry)));
10138                 if (ret) {
10139                         btrfs_abort_transaction(trans, ret);
10140                         goto out_fail;
10141                 }
10142         }
10143
10144         ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
10145                              new_dentry->d_name.name,
10146                              new_dentry->d_name.len, 0, index);
10147         if (ret) {
10148                 btrfs_abort_transaction(trans, ret);
10149                 goto out_fail;
10150         }
10151
10152         if (old_inode->i_nlink == 1)
10153                 BTRFS_I(old_inode)->dir_index = index;
10154
10155         if (log_pinned) {
10156                 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
10157                                    new_dentry->d_parent);
10158                 btrfs_end_log_trans(root);
10159                 log_pinned = false;
10160         }
10161
10162         if (flags & RENAME_WHITEOUT) {
10163                 ret = btrfs_whiteout_for_rename(trans, root, old_dir,
10164                                                 old_dentry);
10165
10166                 if (ret) {
10167                         btrfs_abort_transaction(trans, ret);
10168                         goto out_fail;
10169                 }
10170         }
10171 out_fail:
10172         /*
10173          * If we have pinned the log and an error happened, we unpin tasks
10174          * trying to sync the log and force them to fallback to a transaction
10175          * commit if the log currently contains any of the inodes involved in
10176          * this rename operation (to ensure we do not persist a log with an
10177          * inconsistent state for any of these inodes or leading to any
10178          * inconsistencies when replayed). If the transaction was aborted, the
10179          * abortion reason is propagated to userspace when attempting to commit
10180          * the transaction. If the log does not contain any of these inodes, we
10181          * allow the tasks to sync it.
10182          */
10183         if (ret && log_pinned) {
10184                 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
10185                     btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
10186                     btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
10187                     (new_inode &&
10188                      btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
10189                         btrfs_set_log_full_commit(trans);
10190
10191                 btrfs_end_log_trans(root);
10192                 log_pinned = false;
10193         }
10194         ret2 = btrfs_end_transaction(trans);
10195         ret = ret ? ret : ret2;
10196 out_notrans:
10197         if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
10198                 up_read(&fs_info->subvol_sem);
10199
10200         return ret;
10201 }
10202
10203 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry,
10204                          struct inode *new_dir, struct dentry *new_dentry,
10205                          unsigned int flags)
10206 {
10207         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
10208                 return -EINVAL;
10209
10210         if (flags & RENAME_EXCHANGE)
10211                 return btrfs_rename_exchange(old_dir, old_dentry, new_dir,
10212                                           new_dentry);
10213
10214         return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags);
10215 }
10216
10217 struct btrfs_delalloc_work {
10218         struct inode *inode;
10219         struct completion completion;
10220         struct list_head list;
10221         struct btrfs_work work;
10222 };
10223
10224 static void btrfs_run_delalloc_work(struct btrfs_work *work)
10225 {
10226         struct btrfs_delalloc_work *delalloc_work;
10227         struct inode *inode;
10228
10229         delalloc_work = container_of(work, struct btrfs_delalloc_work,
10230                                      work);
10231         inode = delalloc_work->inode;
10232         filemap_flush(inode->i_mapping);
10233         if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
10234                                 &BTRFS_I(inode)->runtime_flags))
10235                 filemap_flush(inode->i_mapping);
10236
10237         iput(inode);
10238         complete(&delalloc_work->completion);
10239 }
10240
10241 static struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode)
10242 {
10243         struct btrfs_delalloc_work *work;
10244
10245         work = kmalloc(sizeof(*work), GFP_NOFS);
10246         if (!work)
10247                 return NULL;
10248
10249         init_completion(&work->completion);
10250         INIT_LIST_HEAD(&work->list);
10251         work->inode = inode;
10252         btrfs_init_work(&work->work, btrfs_run_delalloc_work, NULL, NULL);
10253
10254         return work;
10255 }
10256
10257 /*
10258  * some fairly slow code that needs optimization. This walks the list
10259  * of all the inodes with pending delalloc and forces them to disk.
10260  */
10261 static int start_delalloc_inodes(struct btrfs_root *root, int nr, bool snapshot)
10262 {
10263         struct btrfs_inode *binode;
10264         struct inode *inode;
10265         struct btrfs_delalloc_work *work, *next;
10266         struct list_head works;
10267         struct list_head splice;
10268         int ret = 0;
10269
10270         INIT_LIST_HEAD(&works);
10271         INIT_LIST_HEAD(&splice);
10272
10273         mutex_lock(&root->delalloc_mutex);
10274         spin_lock(&root->delalloc_lock);
10275         list_splice_init(&root->delalloc_inodes, &splice);
10276         while (!list_empty(&splice)) {
10277                 binode = list_entry(splice.next, struct btrfs_inode,
10278                                     delalloc_inodes);
10279
10280                 list_move_tail(&binode->delalloc_inodes,
10281                                &root->delalloc_inodes);
10282                 inode = igrab(&binode->vfs_inode);
10283                 if (!inode) {
10284                         cond_resched_lock(&root->delalloc_lock);
10285                         continue;
10286                 }
10287                 spin_unlock(&root->delalloc_lock);
10288
10289                 if (snapshot)
10290                         set_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
10291                                 &binode->runtime_flags);
10292                 work = btrfs_alloc_delalloc_work(inode);
10293                 if (!work) {
10294                         iput(inode);
10295                         ret = -ENOMEM;
10296                         goto out;
10297                 }
10298                 list_add_tail(&work->list, &works);
10299                 btrfs_queue_work(root->fs_info->flush_workers,
10300                                  &work->work);
10301                 ret++;
10302                 if (nr != -1 && ret >= nr)
10303                         goto out;
10304                 cond_resched();
10305                 spin_lock(&root->delalloc_lock);
10306         }
10307         spin_unlock(&root->delalloc_lock);
10308
10309 out:
10310         list_for_each_entry_safe(work, next, &works, list) {
10311                 list_del_init(&work->list);
10312                 wait_for_completion(&work->completion);
10313                 kfree(work);
10314         }
10315
10316         if (!list_empty(&splice)) {
10317                 spin_lock(&root->delalloc_lock);
10318                 list_splice_tail(&splice, &root->delalloc_inodes);
10319                 spin_unlock(&root->delalloc_lock);
10320         }
10321         mutex_unlock(&root->delalloc_mutex);
10322         return ret;
10323 }
10324
10325 int btrfs_start_delalloc_snapshot(struct btrfs_root *root)
10326 {
10327         struct btrfs_fs_info *fs_info = root->fs_info;
10328         int ret;
10329
10330         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
10331                 return -EROFS;
10332
10333         ret = start_delalloc_inodes(root, -1, true);
10334         if (ret > 0)
10335                 ret = 0;
10336         return ret;
10337 }
10338
10339 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int nr)
10340 {
10341         struct btrfs_root *root;
10342         struct list_head splice;
10343         int ret;
10344
10345         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
10346                 return -EROFS;
10347
10348         INIT_LIST_HEAD(&splice);
10349
10350         mutex_lock(&fs_info->delalloc_root_mutex);
10351         spin_lock(&fs_info->delalloc_root_lock);
10352         list_splice_init(&fs_info->delalloc_roots, &splice);
10353         while (!list_empty(&splice) && nr) {
10354                 root = list_first_entry(&splice, struct btrfs_root,
10355                                         delalloc_root);
10356                 root = btrfs_grab_fs_root(root);
10357                 BUG_ON(!root);
10358                 list_move_tail(&root->delalloc_root,
10359                                &fs_info->delalloc_roots);
10360                 spin_unlock(&fs_info->delalloc_root_lock);
10361
10362                 ret = start_delalloc_inodes(root, nr, false);
10363                 btrfs_put_fs_root(root);
10364                 if (ret < 0)
10365                         goto out;
10366
10367                 if (nr != -1) {
10368                         nr -= ret;
10369                         WARN_ON(nr < 0);
10370                 }
10371                 spin_lock(&fs_info->delalloc_root_lock);
10372         }
10373         spin_unlock(&fs_info->delalloc_root_lock);
10374
10375         ret = 0;
10376 out:
10377         if (!list_empty(&splice)) {
10378                 spin_lock(&fs_info->delalloc_root_lock);
10379                 list_splice_tail(&splice, &fs_info->delalloc_roots);
10380                 spin_unlock(&fs_info->delalloc_root_lock);
10381         }
10382         mutex_unlock(&fs_info->delalloc_root_mutex);
10383         return ret;
10384 }
10385
10386 static int btrfs_symlink(struct inode *dir, struct dentry *dentry,
10387                          const char *symname)
10388 {
10389         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
10390         struct btrfs_trans_handle *trans;
10391         struct btrfs_root *root = BTRFS_I(dir)->root;
10392         struct btrfs_path *path;
10393         struct btrfs_key key;
10394         struct inode *inode = NULL;
10395         int err;
10396         u64 objectid;
10397         u64 index = 0;
10398         int name_len;
10399         int datasize;
10400         unsigned long ptr;
10401         struct btrfs_file_extent_item *ei;
10402         struct extent_buffer *leaf;
10403
10404         name_len = strlen(symname);
10405         if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))
10406                 return -ENAMETOOLONG;
10407
10408         /*
10409          * 2 items for inode item and ref
10410          * 2 items for dir items
10411          * 1 item for updating parent inode item
10412          * 1 item for the inline extent item
10413          * 1 item for xattr if selinux is on
10414          */
10415         trans = btrfs_start_transaction(root, 7);
10416         if (IS_ERR(trans))
10417                 return PTR_ERR(trans);
10418
10419         err = btrfs_find_free_objectid(root, &objectid);
10420         if (err)
10421                 goto out_unlock;
10422
10423         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
10424                                 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)),
10425                                 objectid, S_IFLNK|S_IRWXUGO, &index);
10426         if (IS_ERR(inode)) {
10427                 err = PTR_ERR(inode);
10428                 inode = NULL;
10429                 goto out_unlock;
10430         }
10431
10432         /*
10433         * If the active LSM wants to access the inode during
10434         * d_instantiate it needs these. Smack checks to see
10435         * if the filesystem supports xattrs by looking at the
10436         * ops vector.
10437         */
10438         inode->i_fop = &btrfs_file_operations;
10439         inode->i_op = &btrfs_file_inode_operations;
10440         inode->i_mapping->a_ops = &btrfs_aops;
10441         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
10442
10443         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
10444         if (err)
10445                 goto out_unlock;
10446
10447         path = btrfs_alloc_path();
10448         if (!path) {
10449                 err = -ENOMEM;
10450                 goto out_unlock;
10451         }
10452         key.objectid = btrfs_ino(BTRFS_I(inode));
10453         key.offset = 0;
10454         key.type = BTRFS_EXTENT_DATA_KEY;
10455         datasize = btrfs_file_extent_calc_inline_size(name_len);
10456         err = btrfs_insert_empty_item(trans, root, path, &key,
10457                                       datasize);
10458         if (err) {
10459                 btrfs_free_path(path);
10460                 goto out_unlock;
10461         }
10462         leaf = path->nodes[0];
10463         ei = btrfs_item_ptr(leaf, path->slots[0],
10464                             struct btrfs_file_extent_item);
10465         btrfs_set_file_extent_generation(leaf, ei, trans->transid);
10466         btrfs_set_file_extent_type(leaf, ei,
10467                                    BTRFS_FILE_EXTENT_INLINE);
10468         btrfs_set_file_extent_encryption(leaf, ei, 0);
10469         btrfs_set_file_extent_compression(leaf, ei, 0);
10470         btrfs_set_file_extent_other_encoding(leaf, ei, 0);
10471         btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);
10472
10473         ptr = btrfs_file_extent_inline_start(ei);
10474         write_extent_buffer(leaf, symname, ptr, name_len);
10475         btrfs_mark_buffer_dirty(leaf);
10476         btrfs_free_path(path);
10477
10478         inode->i_op = &btrfs_symlink_inode_operations;
10479         inode_nohighmem(inode);
10480         inode_set_bytes(inode, name_len);
10481         btrfs_i_size_write(BTRFS_I(inode), name_len);
10482         err = btrfs_update_inode(trans, root, inode);
10483         /*
10484          * Last step, add directory indexes for our symlink inode. This is the
10485          * last step to avoid extra cleanup of these indexes if an error happens
10486          * elsewhere above.
10487          */
10488         if (!err)
10489                 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
10490                                 BTRFS_I(inode), 0, index);
10491         if (err)
10492                 goto out_unlock;
10493
10494         d_instantiate_new(dentry, inode);
10495
10496 out_unlock:
10497         btrfs_end_transaction(trans);
10498         if (err && inode) {
10499                 inode_dec_link_count(inode);
10500                 discard_new_inode(inode);
10501         }
10502         btrfs_btree_balance_dirty(fs_info);
10503         return err;
10504 }
10505
10506 static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
10507                                        u64 start, u64 num_bytes, u64 min_size,
10508                                        loff_t actual_len, u64 *alloc_hint,
10509                                        struct btrfs_trans_handle *trans)
10510 {
10511         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
10512         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
10513         struct extent_map *em;
10514         struct btrfs_root *root = BTRFS_I(inode)->root;
10515         struct btrfs_key ins;
10516         u64 cur_offset = start;
10517         u64 clear_offset = start;
10518         u64 i_size;
10519         u64 cur_bytes;
10520         u64 last_alloc = (u64)-1;
10521         int ret = 0;
10522         bool own_trans = true;
10523         u64 end = start + num_bytes - 1;
10524
10525         if (trans)
10526                 own_trans = false;
10527         while (num_bytes > 0) {
10528                 if (own_trans) {
10529                         trans = btrfs_start_transaction(root, 3);
10530                         if (IS_ERR(trans)) {
10531                                 ret = PTR_ERR(trans);
10532                                 break;
10533                         }
10534                 }
10535
10536                 cur_bytes = min_t(u64, num_bytes, SZ_256M);
10537                 cur_bytes = max(cur_bytes, min_size);
10538                 /*
10539                  * If we are severely fragmented we could end up with really
10540                  * small allocations, so if the allocator is returning small
10541                  * chunks lets make its job easier by only searching for those
10542                  * sized chunks.
10543                  */
10544                 cur_bytes = min(cur_bytes, last_alloc);
10545                 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
10546                                 min_size, 0, *alloc_hint, &ins, 1, 0);
10547                 if (ret) {
10548                         if (own_trans)
10549                                 btrfs_end_transaction(trans);
10550                         break;
10551                 }
10552
10553                 /*
10554                  * We've reserved this space, and thus converted it from
10555                  * ->bytes_may_use to ->bytes_reserved.  Any error that happens
10556                  * from here on out we will only need to clear our reservation
10557                  * for the remaining unreserved area, so advance our
10558                  * clear_offset by our extent size.
10559                  */
10560                 clear_offset += ins.offset;
10561                 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
10562
10563                 last_alloc = ins.offset;
10564                 ret = insert_reserved_file_extent(trans, inode,
10565                                                   cur_offset, ins.objectid,
10566                                                   ins.offset, ins.offset,
10567                                                   ins.offset, 0, 0, 0,
10568                                                   BTRFS_FILE_EXTENT_PREALLOC);
10569                 if (ret) {
10570                         btrfs_free_reserved_extent(fs_info, ins.objectid,
10571                                                    ins.offset, 0);
10572                         btrfs_abort_transaction(trans, ret);
10573                         if (own_trans)
10574                                 btrfs_end_transaction(trans);
10575                         break;
10576                 }
10577
10578                 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
10579                                         cur_offset + ins.offset -1, 0);
10580
10581                 em = alloc_extent_map();
10582                 if (!em) {
10583                         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
10584                                 &BTRFS_I(inode)->runtime_flags);
10585                         goto next;
10586                 }
10587
10588                 em->start = cur_offset;
10589                 em->orig_start = cur_offset;
10590                 em->len = ins.offset;
10591                 em->block_start = ins.objectid;
10592                 em->block_len = ins.offset;
10593                 em->orig_block_len = ins.offset;
10594                 em->ram_bytes = ins.offset;
10595                 em->bdev = fs_info->fs_devices->latest_bdev;
10596                 set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
10597                 em->generation = trans->transid;
10598
10599                 while (1) {
10600                         write_lock(&em_tree->lock);
10601                         ret = add_extent_mapping(em_tree, em, 1);
10602                         write_unlock(&em_tree->lock);
10603                         if (ret != -EEXIST)
10604                                 break;
10605                         btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
10606                                                 cur_offset + ins.offset - 1,
10607                                                 0);
10608                 }
10609                 free_extent_map(em);
10610 next:
10611                 num_bytes -= ins.offset;
10612                 cur_offset += ins.offset;
10613                 *alloc_hint = ins.objectid + ins.offset;
10614
10615                 inode_inc_iversion(inode);
10616                 inode->i_ctime = current_time(inode);
10617                 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;
10618                 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
10619                     (actual_len > inode->i_size) &&
10620                     (cur_offset > inode->i_size)) {
10621                         if (cur_offset > actual_len)
10622                                 i_size = actual_len;
10623                         else
10624                                 i_size = cur_offset;
10625                         i_size_write(inode, i_size);
10626                         btrfs_ordered_update_i_size(inode, i_size, NULL);
10627                 }
10628
10629                 ret = btrfs_update_inode(trans, root, inode);
10630
10631                 if (ret) {
10632                         btrfs_abort_transaction(trans, ret);
10633                         if (own_trans)
10634                                 btrfs_end_transaction(trans);
10635                         break;
10636                 }
10637
10638                 if (own_trans)
10639                         btrfs_end_transaction(trans);
10640         }
10641         if (clear_offset < end)
10642                 btrfs_free_reserved_data_space(inode, NULL, clear_offset,
10643                         end - clear_offset + 1);
10644         return ret;
10645 }
10646
10647 int btrfs_prealloc_file_range(struct inode *inode, int mode,
10648                               u64 start, u64 num_bytes, u64 min_size,
10649                               loff_t actual_len, u64 *alloc_hint)
10650 {
10651         return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
10652                                            min_size, actual_len, alloc_hint,
10653                                            NULL);
10654 }
10655
10656 int btrfs_prealloc_file_range_trans(struct inode *inode,
10657                                     struct btrfs_trans_handle *trans, int mode,
10658                                     u64 start, u64 num_bytes, u64 min_size,
10659                                     loff_t actual_len, u64 *alloc_hint)
10660 {
10661         return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
10662                                            min_size, actual_len, alloc_hint, trans);
10663 }
10664
10665 static int btrfs_set_page_dirty(struct page *page)
10666 {
10667         return __set_page_dirty_nobuffers(page);
10668 }
10669
10670 static int btrfs_permission(struct inode *inode, int mask)
10671 {
10672         struct btrfs_root *root = BTRFS_I(inode)->root;
10673         umode_t mode = inode->i_mode;
10674
10675         if (mask & MAY_WRITE &&
10676             (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {
10677                 if (btrfs_root_readonly(root))
10678                         return -EROFS;
10679                 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
10680                         return -EACCES;
10681         }
10682         return generic_permission(inode, mask);
10683 }
10684
10685 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
10686 {
10687         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
10688         struct btrfs_trans_handle *trans;
10689         struct btrfs_root *root = BTRFS_I(dir)->root;
10690         struct inode *inode = NULL;
10691         u64 objectid;
10692         u64 index;
10693         int ret = 0;
10694
10695         /*
10696          * 5 units required for adding orphan entry
10697          */
10698         trans = btrfs_start_transaction(root, 5);
10699         if (IS_ERR(trans))
10700                 return PTR_ERR(trans);
10701
10702         ret = btrfs_find_free_objectid(root, &objectid);
10703         if (ret)
10704                 goto out;
10705
10706         inode = btrfs_new_inode(trans, root, dir, NULL, 0,
10707                         btrfs_ino(BTRFS_I(dir)), objectid, mode, &index);
10708         if (IS_ERR(inode)) {
10709                 ret = PTR_ERR(inode);
10710                 inode = NULL;
10711                 goto out;
10712         }
10713
10714         inode->i_fop = &btrfs_file_operations;
10715         inode->i_op = &btrfs_file_inode_operations;
10716
10717         inode->i_mapping->a_ops = &btrfs_aops;
10718         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
10719
10720         ret = btrfs_init_inode_security(trans, inode, dir, NULL);
10721         if (ret)
10722                 goto out;
10723
10724         ret = btrfs_update_inode(trans, root, inode);
10725         if (ret)
10726                 goto out;
10727         ret = btrfs_orphan_add(trans, BTRFS_I(inode));
10728         if (ret)
10729                 goto out;
10730
10731         /*
10732          * We set number of links to 0 in btrfs_new_inode(), and here we set
10733          * it to 1 because d_tmpfile() will issue a warning if the count is 0,
10734          * through:
10735          *
10736          *    d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
10737          */
10738         set_nlink(inode, 1);
10739         d_tmpfile(dentry, inode);
10740         unlock_new_inode(inode);
10741         mark_inode_dirty(inode);
10742 out:
10743         btrfs_end_transaction(trans);
10744         if (ret && inode)
10745                 discard_new_inode(inode);
10746         btrfs_btree_balance_dirty(fs_info);
10747         return ret;
10748 }
10749
10750 void btrfs_set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
10751 {
10752         struct inode *inode = tree->private_data;
10753         unsigned long index = start >> PAGE_SHIFT;
10754         unsigned long end_index = end >> PAGE_SHIFT;
10755         struct page *page;
10756
10757         while (index <= end_index) {
10758                 page = find_get_page(inode->i_mapping, index);
10759                 ASSERT(page); /* Pages should be in the extent_io_tree */
10760                 set_page_writeback(page);
10761                 put_page(page);
10762                 index++;
10763         }
10764 }
10765
10766 #ifdef CONFIG_SWAP
10767 /*
10768  * Add an entry indicating a block group or device which is pinned by a
10769  * swapfile. Returns 0 on success, 1 if there is already an entry for it, or a
10770  * negative errno on failure.
10771  */
10772 static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr,
10773                                   bool is_block_group)
10774 {
10775         struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
10776         struct btrfs_swapfile_pin *sp, *entry;
10777         struct rb_node **p;
10778         struct rb_node *parent = NULL;
10779
10780         sp = kmalloc(sizeof(*sp), GFP_NOFS);
10781         if (!sp)
10782                 return -ENOMEM;
10783         sp->ptr = ptr;
10784         sp->inode = inode;
10785         sp->is_block_group = is_block_group;
10786
10787         spin_lock(&fs_info->swapfile_pins_lock);
10788         p = &fs_info->swapfile_pins.rb_node;
10789         while (*p) {
10790                 parent = *p;
10791                 entry = rb_entry(parent, struct btrfs_swapfile_pin, node);
10792                 if (sp->ptr < entry->ptr ||
10793                     (sp->ptr == entry->ptr && sp->inode < entry->inode)) {
10794                         p = &(*p)->rb_left;
10795                 } else if (sp->ptr > entry->ptr ||
10796                            (sp->ptr == entry->ptr && sp->inode > entry->inode)) {
10797                         p = &(*p)->rb_right;
10798                 } else {
10799                         spin_unlock(&fs_info->swapfile_pins_lock);
10800                         kfree(sp);
10801                         return 1;
10802                 }
10803         }
10804         rb_link_node(&sp->node, parent, p);
10805         rb_insert_color(&sp->node, &fs_info->swapfile_pins);
10806         spin_unlock(&fs_info->swapfile_pins_lock);
10807         return 0;
10808 }
10809
10810 /* Free all of the entries pinned by this swapfile. */
10811 static void btrfs_free_swapfile_pins(struct inode *inode)
10812 {
10813         struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
10814         struct btrfs_swapfile_pin *sp;
10815         struct rb_node *node, *next;
10816
10817         spin_lock(&fs_info->swapfile_pins_lock);
10818         node = rb_first(&fs_info->swapfile_pins);
10819         while (node) {
10820                 next = rb_next(node);
10821                 sp = rb_entry(node, struct btrfs_swapfile_pin, node);
10822                 if (sp->inode == inode) {
10823                         rb_erase(&sp->node, &fs_info->swapfile_pins);
10824                         if (sp->is_block_group)
10825                                 btrfs_put_block_group(sp->ptr);
10826                         kfree(sp);
10827                 }
10828                 node = next;
10829         }
10830         spin_unlock(&fs_info->swapfile_pins_lock);
10831 }
10832
10833 struct btrfs_swap_info {
10834         u64 start;
10835         u64 block_start;
10836         u64 block_len;
10837         u64 lowest_ppage;
10838         u64 highest_ppage;
10839         unsigned long nr_pages;
10840         int nr_extents;
10841 };
10842
10843 static int btrfs_add_swap_extent(struct swap_info_struct *sis,
10844                                  struct btrfs_swap_info *bsi)
10845 {
10846         unsigned long nr_pages;
10847         unsigned long max_pages;
10848         u64 first_ppage, first_ppage_reported, next_ppage;
10849         int ret;
10850
10851         /*
10852          * Our swapfile may have had its size extended after the swap header was
10853          * written. In that case activating the swapfile should not go beyond
10854          * the max size set in the swap header.
10855          */
10856         if (bsi->nr_pages >= sis->max)
10857                 return 0;
10858
10859         max_pages = sis->max - bsi->nr_pages;
10860         first_ppage = ALIGN(bsi->block_start, PAGE_SIZE) >> PAGE_SHIFT;
10861         next_ppage = ALIGN_DOWN(bsi->block_start + bsi->block_len,
10862                                 PAGE_SIZE) >> PAGE_SHIFT;
10863
10864         if (first_ppage >= next_ppage)
10865                 return 0;
10866         nr_pages = next_ppage - first_ppage;
10867         nr_pages = min(nr_pages, max_pages);
10868
10869         first_ppage_reported = first_ppage;
10870         if (bsi->start == 0)
10871                 first_ppage_reported++;
10872         if (bsi->lowest_ppage > first_ppage_reported)
10873                 bsi->lowest_ppage = first_ppage_reported;
10874         if (bsi->highest_ppage < (next_ppage - 1))
10875                 bsi->highest_ppage = next_ppage - 1;
10876
10877         ret = add_swap_extent(sis, bsi->nr_pages, nr_pages, first_ppage);
10878         if (ret < 0)
10879                 return ret;
10880         bsi->nr_extents += ret;
10881         bsi->nr_pages += nr_pages;
10882         return 0;
10883 }
10884
10885 static void btrfs_swap_deactivate(struct file *file)
10886 {
10887         struct inode *inode = file_inode(file);
10888
10889         btrfs_free_swapfile_pins(inode);
10890         atomic_dec(&BTRFS_I(inode)->root->nr_swapfiles);
10891 }
10892
10893 static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file,
10894                                sector_t *span)
10895 {
10896         struct inode *inode = file_inode(file);
10897         struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
10898         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
10899         struct extent_state *cached_state = NULL;
10900         struct extent_map *em = NULL;
10901         struct btrfs_device *device = NULL;
10902         struct btrfs_swap_info bsi = {
10903                 .lowest_ppage = (sector_t)-1ULL,
10904         };
10905         int ret = 0;
10906         u64 isize;
10907         u64 start;
10908
10909         /*
10910          * If the swap file was just created, make sure delalloc is done. If the
10911          * file changes again after this, the user is doing something stupid and
10912          * we don't really care.
10913          */
10914         ret = btrfs_wait_ordered_range(inode, 0, (u64)-1);
10915         if (ret)
10916                 return ret;
10917
10918         /*
10919          * The inode is locked, so these flags won't change after we check them.
10920          */
10921         if (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) {
10922                 btrfs_warn(fs_info, "swapfile must not be compressed");
10923                 return -EINVAL;
10924         }
10925         if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW)) {
10926                 btrfs_warn(fs_info, "swapfile must not be copy-on-write");
10927                 return -EINVAL;
10928         }
10929         if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) {
10930                 btrfs_warn(fs_info, "swapfile must not be checksummed");
10931                 return -EINVAL;
10932         }
10933
10934         /*
10935          * Balance or device remove/replace/resize can move stuff around from
10936          * under us. The EXCL_OP flag makes sure they aren't running/won't run
10937          * concurrently while we are mapping the swap extents, and
10938          * fs_info->swapfile_pins prevents them from running while the swap file
10939          * is active and moving the extents. Note that this also prevents a
10940          * concurrent device add which isn't actually necessary, but it's not
10941          * really worth the trouble to allow it.
10942          */
10943         if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags)) {
10944                 btrfs_warn(fs_info,
10945            "cannot activate swapfile while exclusive operation is running");
10946                 return -EBUSY;
10947         }
10948         /*
10949          * Snapshots can create extents which require COW even if NODATACOW is
10950          * set. We use this counter to prevent snapshots. We must increment it
10951          * before walking the extents because we don't want a concurrent
10952          * snapshot to run after we've already checked the extents.
10953          */
10954         atomic_inc(&BTRFS_I(inode)->root->nr_swapfiles);
10955
10956         isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize);
10957
10958         lock_extent_bits(io_tree, 0, isize - 1, &cached_state);
10959         start = 0;
10960         while (start < isize) {
10961                 u64 logical_block_start, physical_block_start;
10962                 struct btrfs_block_group_cache *bg;
10963                 u64 len = isize - start;
10964
10965                 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len, 0);
10966                 if (IS_ERR(em)) {
10967                         ret = PTR_ERR(em);
10968                         goto out;
10969                 }
10970
10971                 if (em->block_start == EXTENT_MAP_HOLE) {
10972                         btrfs_warn(fs_info, "swapfile must not have holes");
10973                         ret = -EINVAL;
10974                         goto out;
10975                 }
10976                 if (em->block_start == EXTENT_MAP_INLINE) {
10977                         /*
10978                          * It's unlikely we'll ever actually find ourselves
10979                          * here, as a file small enough to fit inline won't be
10980                          * big enough to store more than the swap header, but in
10981                          * case something changes in the future, let's catch it
10982                          * here rather than later.
10983                          */
10984                         btrfs_warn(fs_info, "swapfile must not be inline");
10985                         ret = -EINVAL;
10986                         goto out;
10987                 }
10988                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
10989                         btrfs_warn(fs_info, "swapfile must not be compressed");
10990                         ret = -EINVAL;
10991                         goto out;
10992                 }
10993
10994                 logical_block_start = em->block_start + (start - em->start);
10995                 len = min(len, em->len - (start - em->start));
10996                 free_extent_map(em);
10997                 em = NULL;
10998
10999                 ret = can_nocow_extent(inode, start, &len, NULL, NULL, NULL, true);
11000                 if (ret < 0) {
11001                         goto out;
11002                 } else if (ret) {
11003                         ret = 0;
11004                 } else {
11005                         btrfs_warn(fs_info,
11006                                    "swapfile must not be copy-on-write");
11007                         ret = -EINVAL;
11008                         goto out;
11009                 }
11010
11011                 em = btrfs_get_chunk_map(fs_info, logical_block_start, len);
11012                 if (IS_ERR(em)) {
11013                         ret = PTR_ERR(em);
11014                         goto out;
11015                 }
11016
11017                 if (em->map_lookup->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
11018                         btrfs_warn(fs_info,
11019                                    "swapfile must have single data profile");
11020                         ret = -EINVAL;
11021                         goto out;
11022                 }
11023
11024                 if (device == NULL) {
11025                         device = em->map_lookup->stripes[0].dev;
11026                         ret = btrfs_add_swapfile_pin(inode, device, false);
11027                         if (ret == 1)
11028                                 ret = 0;
11029                         else if (ret)
11030                                 goto out;
11031                 } else if (device != em->map_lookup->stripes[0].dev) {
11032                         btrfs_warn(fs_info, "swapfile must be on one device");
11033                         ret = -EINVAL;
11034                         goto out;
11035                 }
11036
11037                 physical_block_start = (em->map_lookup->stripes[0].physical +
11038                                         (logical_block_start - em->start));
11039                 len = min(len, em->len - (logical_block_start - em->start));
11040                 free_extent_map(em);
11041                 em = NULL;
11042
11043                 bg = btrfs_lookup_block_group(fs_info, logical_block_start);
11044                 if (!bg) {
11045                         btrfs_warn(fs_info,
11046                            "could not find block group containing swapfile");
11047                         ret = -EINVAL;
11048                         goto out;
11049                 }
11050
11051                 ret = btrfs_add_swapfile_pin(inode, bg, true);
11052                 if (ret) {
11053                         btrfs_put_block_group(bg);
11054                         if (ret == 1)
11055                                 ret = 0;
11056                         else
11057                                 goto out;
11058                 }
11059
11060                 if (bsi.block_len &&
11061                     bsi.block_start + bsi.block_len == physical_block_start) {
11062                         bsi.block_len += len;
11063                 } else {
11064                         if (bsi.block_len) {
11065                                 ret = btrfs_add_swap_extent(sis, &bsi);
11066                                 if (ret)
11067                                         goto out;
11068                         }
11069                         bsi.start = start;
11070                         bsi.block_start = physical_block_start;
11071                         bsi.block_len = len;
11072                 }
11073
11074                 start += len;
11075         }
11076
11077         if (bsi.block_len)
11078                 ret = btrfs_add_swap_extent(sis, &bsi);
11079
11080 out:
11081         if (!IS_ERR_OR_NULL(em))
11082                 free_extent_map(em);
11083
11084         unlock_extent_cached(io_tree, 0, isize - 1, &cached_state);
11085
11086         if (ret)
11087                 btrfs_swap_deactivate(file);
11088
11089         clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
11090
11091         if (ret)
11092                 return ret;
11093
11094         if (device)
11095                 sis->bdev = device->bdev;
11096         *span = bsi.highest_ppage - bsi.lowest_ppage + 1;
11097         sis->max = bsi.nr_pages;
11098         sis->pages = bsi.nr_pages - 1;
11099         sis->highest_bit = bsi.nr_pages - 1;
11100         return bsi.nr_extents;
11101 }
11102 #else
11103 static void btrfs_swap_deactivate(struct file *file)
11104 {
11105 }
11106
11107 static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file,
11108                                sector_t *span)
11109 {
11110         return -EOPNOTSUPP;
11111 }
11112 #endif
11113
11114 static const struct inode_operations btrfs_dir_inode_operations = {
11115         .getattr        = btrfs_getattr,
11116         .lookup         = btrfs_lookup,
11117         .create         = btrfs_create,
11118         .unlink         = btrfs_unlink,
11119         .link           = btrfs_link,
11120         .mkdir          = btrfs_mkdir,
11121         .rmdir          = btrfs_rmdir,
11122         .rename         = btrfs_rename2,
11123         .symlink        = btrfs_symlink,
11124         .setattr        = btrfs_setattr,
11125         .mknod          = btrfs_mknod,
11126         .listxattr      = btrfs_listxattr,
11127         .permission     = btrfs_permission,
11128         .get_acl        = btrfs_get_acl,
11129         .set_acl        = btrfs_set_acl,
11130         .update_time    = btrfs_update_time,
11131         .tmpfile        = btrfs_tmpfile,
11132 };
11133 static const struct inode_operations btrfs_dir_ro_inode_operations = {
11134         .lookup         = btrfs_lookup,
11135         .permission     = btrfs_permission,
11136         .update_time    = btrfs_update_time,
11137 };
11138
11139 static const struct file_operations btrfs_dir_file_operations = {
11140         .llseek         = generic_file_llseek,
11141         .read           = generic_read_dir,
11142         .iterate_shared = btrfs_real_readdir,
11143         .open           = btrfs_opendir,
11144         .unlocked_ioctl = btrfs_ioctl,
11145 #ifdef CONFIG_COMPAT
11146         .compat_ioctl   = btrfs_compat_ioctl,
11147 #endif
11148         .release        = btrfs_release_file,
11149         .fsync          = btrfs_sync_file,
11150 };
11151
11152 static const struct extent_io_ops btrfs_extent_io_ops = {
11153         /* mandatory callbacks */
11154         .submit_bio_hook = btrfs_submit_bio_hook,
11155         .readpage_end_io_hook = btrfs_readpage_end_io_hook,
11156 };
11157
11158 /*
11159  * btrfs doesn't support the bmap operation because swapfiles
11160  * use bmap to make a mapping of extents in the file.  They assume
11161  * these extents won't change over the life of the file and they
11162  * use the bmap result to do IO directly to the drive.
11163  *
11164  * the btrfs bmap call would return logical addresses that aren't
11165  * suitable for IO and they also will change frequently as COW
11166  * operations happen.  So, swapfile + btrfs == corruption.
11167  *
11168  * For now we're avoiding this by dropping bmap.
11169  */
11170 static const struct address_space_operations btrfs_aops = {
11171         .readpage       = btrfs_readpage,
11172         .writepage      = btrfs_writepage,
11173         .writepages     = btrfs_writepages,
11174         .readpages      = btrfs_readpages,
11175         .direct_IO      = btrfs_direct_IO,
11176         .invalidatepage = btrfs_invalidatepage,
11177         .releasepage    = btrfs_releasepage,
11178         .set_page_dirty = btrfs_set_page_dirty,
11179         .error_remove_page = generic_error_remove_page,
11180         .swap_activate  = btrfs_swap_activate,
11181         .swap_deactivate = btrfs_swap_deactivate,
11182 };
11183
11184 static const struct inode_operations btrfs_file_inode_operations = {
11185         .getattr        = btrfs_getattr,
11186         .setattr        = btrfs_setattr,
11187         .listxattr      = btrfs_listxattr,
11188         .permission     = btrfs_permission,
11189         .fiemap         = btrfs_fiemap,
11190         .get_acl        = btrfs_get_acl,
11191         .set_acl        = btrfs_set_acl,
11192         .update_time    = btrfs_update_time,
11193 };
11194 static const struct inode_operations btrfs_special_inode_operations = {
11195         .getattr        = btrfs_getattr,
11196         .setattr        = btrfs_setattr,
11197         .permission     = btrfs_permission,
11198         .listxattr      = btrfs_listxattr,
11199         .get_acl        = btrfs_get_acl,
11200         .set_acl        = btrfs_set_acl,
11201         .update_time    = btrfs_update_time,
11202 };
11203 static const struct inode_operations btrfs_symlink_inode_operations = {
11204         .get_link       = page_get_link,
11205         .getattr        = btrfs_getattr,
11206         .setattr        = btrfs_setattr,
11207         .permission     = btrfs_permission,
11208         .listxattr      = btrfs_listxattr,
11209         .update_time    = btrfs_update_time,
11210 };
11211
11212 const struct dentry_operations btrfs_dentry_operations = {
11213         .d_delete       = btrfs_dentry_delete,
11214 };