mm/shmem.c

   1 /*
   2  * Resizable virtual memory filesystem for Linux.
   3  *
   4  * Copyright (C) 2000 Linus Torvalds.
   5  *               2000 Transmeta Corp.
   6  *               2000-2001 Christoph Rohland
   7  *               2000-2001 SAP AG
   8  *               2002 Red Hat Inc.
   9  * Copyright (C) 2002-2011 Hugh Dickins.
  10  * Copyright (C) 2011 Google Inc.
  11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
  12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
  13  *
  14  * Extended attribute support for tmpfs:
  15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
  16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
  17  *
  18  * tiny-shmem:
  19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
  20  *
  21  * This file is released under the GPL.
  22  */
  23
  24 #include <linux/fs.h>
  25 #include <linux/init.h>
  26 #include <linux/vfs.h>
  27 #include <linux/mount.h>
  28 #include <linux/ramfs.h>
  29 #include <linux/pagemap.h>
  30 #include <linux/file.h>
  31 #include <linux/fileattr.h>
  32 #include <linux/mm.h>
  33 #include <linux/random.h>
  34 #include <linux/sched/signal.h>
  35 #include <linux/export.h>
  36 #include <linux/shmem_fs.h>
  37 #include <linux/swap.h>
  38 #include <linux/uio.h>
  39 #include <linux/hugetlb.h>
  40 #include <linux/fs_parser.h>
  41 #include <linux/swapfile.h>
  42 #include <linux/iversion.h>
  43 #include "swap.h"
  44
  45 static struct vfsmount *shm_mnt __ro_after_init;
  46
  47 #ifdef CONFIG_SHMEM
  48 /*
  49  * This virtual memory filesystem is heavily based on the ramfs. It
  50  * extends ramfs by the ability to use swap and honor resource limits
  51  * which makes it a completely usable filesystem.
  52  */
  53
  54 #include <linux/xattr.h>
  55 #include <linux/exportfs.h>
  56 #include <linux/posix_acl.h>
  57 #include <linux/posix_acl_xattr.h>
  58 #include <linux/mman.h>
  59 #include <linux/string.h>
  60 #include <linux/slab.h>
  61 #include <linux/backing-dev.h>
  62 #include <linux/writeback.h>
  63 #include <linux/pagevec.h>
  64 #include <linux/percpu_counter.h>
  65 #include <linux/falloc.h>
  66 #include <linux/splice.h>
  67 #include <linux/security.h>
  68 #include <linux/swapops.h>
  69 #include <linux/mempolicy.h>
  70 #include <linux/namei.h>
  71 #include <linux/ctype.h>
  72 #include <linux/migrate.h>
  73 #include <linux/highmem.h>
  74 #include <linux/seq_file.h>
  75 #include <linux/magic.h>
  76 #include <linux/syscalls.h>
  77 #include <linux/fcntl.h>
  78 #include <uapi/linux/memfd.h>
  79 #include <linux/rmap.h>
  80 #include <linux/uuid.h>
  81 #include <linux/quotaops.h>
  82 #include <linux/rcupdate_wait.h>
  83
  84 #include <linux/uaccess.h>
  85
  86 #include "internal.h"
  87
  88 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
  89 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
  90
  91 /* Pretend that each entry is of this size in directory's i_size */
  92 #define BOGO_DIRENT_SIZE 20
  93
  94 /* Pretend that one inode + its dentry occupy this much memory */
  95 #define BOGO_INODE_SIZE 1024
  96
  97 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
  98 #define SHORT_SYMLINK_LEN 128
  99
 100 /*
 101  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
 102  * inode->i_private (with i_rwsem making sure that it has only one user at
 103  * a time): we would prefer not to enlarge the shmem inode just for that.
 104  */
 105 struct shmem_falloc {
 106         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
 107         pgoff_t start;          /* start of range currently being fallocated */
 108         pgoff_t next;           /* the next page offset to be fallocated */
 109         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
 110         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
 111 };
 112
 113 struct shmem_options {
 114         unsigned long long blocks;
 115         unsigned long long inodes;
 116         struct mempolicy *mpol;
 117         kuid_t uid;
 118         kgid_t gid;
 119         umode_t mode;
 120         bool full_inums;
 121         int huge;
 122         int seen;
 123         bool noswap;
 124         unsigned short quota_types;
 125         struct shmem_quota_limits qlimits;
 126 #define SHMEM_SEEN_BLOCKS 1
 127 #define SHMEM_SEEN_INODES 2
 128 #define SHMEM_SEEN_HUGE 4
 129 #define SHMEM_SEEN_INUMS 8
 130 #define SHMEM_SEEN_NOSWAP 16
 131 #define SHMEM_SEEN_QUOTA 32
 132 };
 133
 134 #ifdef CONFIG_TMPFS
 135 static unsigned long shmem_default_max_blocks(void)
 136 {
 137         return totalram_pages() / 2;
 138 }
 139
 140 static unsigned long shmem_default_max_inodes(void)
 141 {
 142         unsigned long nr_pages = totalram_pages();
 143
 144         return min3(nr_pages - totalhigh_pages(), nr_pages / 2,
 145                         ULONG_MAX / BOGO_INODE_SIZE);
 146 }
 147 #endif
 148
 149 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
 150                         struct folio **foliop, enum sgp_type sgp, gfp_t gfp,
 151                         struct mm_struct *fault_mm, vm_fault_t *fault_type);
 152
 153 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
 154 {
 155         return sb->s_fs_info;
 156 }
 157
 158 /*
 159  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
 160  * for shared memory and for shared anonymous (/dev/zero) mappings
 161  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
 162  * consistent with the pre-accounting of private mappings ...
 163  */
 164 static inline int shmem_acct_size(unsigned long flags, loff_t size)
 165 {
 166         return (flags & VM_NORESERVE) ?
 167                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
 168 }
 169
 170 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
 171 {
 172         if (!(flags & VM_NORESERVE))
 173                 vm_unacct_memory(VM_ACCT(size));
 174 }
 175
 176 static inline int shmem_reacct_size(unsigned long flags,
 177                 loff_t oldsize, loff_t newsize)
 178 {
 179         if (!(flags & VM_NORESERVE)) {
 180                 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
 181                         return security_vm_enough_memory_mm(current->mm,
 182                                         VM_ACCT(newsize) - VM_ACCT(oldsize));
 183                 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
 184                         vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
 185         }
 186         return 0;
 187 }
 188
 189 /*
 190  * ... whereas tmpfs objects are accounted incrementally as
 191  * pages are allocated, in order to allow large sparse files.
 192  * shmem_get_folio reports shmem_acct_blocks failure as -ENOSPC not -ENOMEM,
 193  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
 194  */
 195 static inline int shmem_acct_blocks(unsigned long flags, long pages)
 196 {
 197         if (!(flags & VM_NORESERVE))
 198                 return 0;
 199
 200         return security_vm_enough_memory_mm(current->mm,
 201                         pages * VM_ACCT(PAGE_SIZE));
 202 }
 203
 204 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
 205 {
 206         if (flags & VM_NORESERVE)
 207                 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
 208 }
 209
 210 static int shmem_inode_acct_blocks(struct inode *inode, long pages)
 211 {
 212         struct shmem_inode_info *info = SHMEM_I(inode);
 213         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
 214         int err = -ENOSPC;
 215
 216         if (shmem_acct_blocks(info->flags, pages))
 217                 return err;
 218
 219         might_sleep();  /* when quotas */
 220         if (sbinfo->max_blocks) {
 221                 if (!percpu_counter_limited_add(&sbinfo->used_blocks,
 222                                                 sbinfo->max_blocks, pages))
 223                         goto unacct;
 224
 225                 err = dquot_alloc_block_nodirty(inode, pages);
 226                 if (err) {
 227                         percpu_counter_sub(&sbinfo->used_blocks, pages);
 228                         goto unacct;
 229                 }
 230         } else {
 231                 err = dquot_alloc_block_nodirty(inode, pages);
 232                 if (err)
 233                         goto unacct;
 234         }
 235
 236         return 0;
 237
 238 unacct:
 239         shmem_unacct_blocks(info->flags, pages);
 240         return err;
 241 }
 242
 243 static void shmem_inode_unacct_blocks(struct inode *inode, long pages)
 244 {
 245         struct shmem_inode_info *info = SHMEM_I(inode);
 246         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
 247
 248         might_sleep();  /* when quotas */
 249         dquot_free_block_nodirty(inode, pages);
 250
 251         if (sbinfo->max_blocks)
 252                 percpu_counter_sub(&sbinfo->used_blocks, pages);
 253         shmem_unacct_blocks(info->flags, pages);
 254 }
 255
 256 static const struct super_operations shmem_ops;
 257 const struct address_space_operations shmem_aops;
 258 static const struct file_operations shmem_file_operations;
 259 static const struct inode_operations shmem_inode_operations;
 260 static const struct inode_operations shmem_dir_inode_operations;
 261 static const struct inode_operations shmem_special_inode_operations;
 262 static const struct vm_operations_struct shmem_vm_ops;
 263 static const struct vm_operations_struct shmem_anon_vm_ops;
 264 static struct file_system_type shmem_fs_type;
 265
 266 bool vma_is_anon_shmem(struct vm_area_struct *vma)
 267 {
 268         return vma->vm_ops == &shmem_anon_vm_ops;
 269 }
 270
 271 bool vma_is_shmem(struct vm_area_struct *vma)
 272 {
 273         return vma_is_anon_shmem(vma) || vma->vm_ops == &shmem_vm_ops;
 274 }
 275
 276 static LIST_HEAD(shmem_swaplist);
 277 static DEFINE_MUTEX(shmem_swaplist_mutex);
 278
 279 #ifdef CONFIG_TMPFS_QUOTA
 280
 281 static int shmem_enable_quotas(struct super_block *sb,
 282                                unsigned short quota_types)
 283 {
 284         int type, err = 0;
 285
 286         sb_dqopt(sb)->flags |= DQUOT_QUOTA_SYS_FILE | DQUOT_NOLIST_DIRTY;
 287         for (type = 0; type < SHMEM_MAXQUOTAS; type++) {
 288                 if (!(quota_types & (1 << type)))
 289                         continue;
 290                 err = dquot_load_quota_sb(sb, type, QFMT_SHMEM,
 291                                           DQUOT_USAGE_ENABLED |
 292                                           DQUOT_LIMITS_ENABLED);
 293                 if (err)
 294                         goto out_err;
 295         }
 296         return 0;
 297
 298 out_err:
 299         pr_warn("tmpfs: failed to enable quota tracking (type=%d, err=%d)\n",
 300                 type, err);
 301         for (type--; type >= 0; type--)
 302                 dquot_quota_off(sb, type);
 303         return err;
 304 }
 305
 306 static void shmem_disable_quotas(struct super_block *sb)
 307 {
 308         int type;
 309
 310         for (type = 0; type < SHMEM_MAXQUOTAS; type++)
 311                 dquot_quota_off(sb, type);
 312 }
 313
 314 static struct dquot __rcu **shmem_get_dquots(struct inode *inode)
 315 {
 316         return SHMEM_I(inode)->i_dquot;
 317 }
 318 #endif /* CONFIG_TMPFS_QUOTA */
 319
 320 /*
 321  * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and
 322  * produces a novel ino for the newly allocated inode.
 323  *
 324  * It may also be called when making a hard link to permit the space needed by
 325  * each dentry. However, in that case, no new inode number is needed since that
 326  * internally draws from another pool of inode numbers (currently global
 327  * get_next_ino()). This case is indicated by passing NULL as inop.
 328  */
 329 #define SHMEM_INO_BATCH 1024
 330 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop)
 331 {
 332         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 333         ino_t ino;
 334
 335         if (!(sb->s_flags & SB_KERNMOUNT)) {
 336                 raw_spin_lock(&sbinfo->stat_lock);
 337                 if (sbinfo->max_inodes) {
 338                         if (sbinfo->free_ispace < BOGO_INODE_SIZE) {
 339                                 raw_spin_unlock(&sbinfo->stat_lock);
 340                                 return -ENOSPC;
 341                         }
 342                         sbinfo->free_ispace -= BOGO_INODE_SIZE;
 343                 }
 344                 if (inop) {
 345                         ino = sbinfo->next_ino++;
 346                         if (unlikely(is_zero_ino(ino)))
 347                                 ino = sbinfo->next_ino++;
 348                         if (unlikely(!sbinfo->full_inums &&
 349                                      ino > UINT_MAX)) {
 350                                 /*
 351                                  * Emulate get_next_ino uint wraparound for
 352                                  * compatibility
 353                                  */
 354                                 if (IS_ENABLED(CONFIG_64BIT))
 355                                         pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n",
 356                                                 __func__, MINOR(sb->s_dev));
 357                                 sbinfo->next_ino = 1;
 358                                 ino = sbinfo->next_ino++;
 359                         }
 360                         *inop = ino;
 361                 }
 362                 raw_spin_unlock(&sbinfo->stat_lock);
 363         } else if (inop) {
 364                 /*
 365                  * __shmem_file_setup, one of our callers, is lock-free: it
 366                  * doesn't hold stat_lock in shmem_reserve_inode since
 367                  * max_inodes is always 0, and is called from potentially
 368                  * unknown contexts. As such, use a per-cpu batched allocator
 369                  * which doesn't require the per-sb stat_lock unless we are at
 370                  * the batch boundary.
 371                  *
 372                  * We don't need to worry about inode{32,64} since SB_KERNMOUNT
 373                  * shmem mounts are not exposed to userspace, so we don't need
 374                  * to worry about things like glibc compatibility.
 375                  */
 376                 ino_t *next_ino;
 377
 378                 next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu());
 379                 ino = *next_ino;
 380                 if (unlikely(ino % SHMEM_INO_BATCH == 0)) {
 381                         raw_spin_lock(&sbinfo->stat_lock);
 382                         ino = sbinfo->next_ino;
 383                         sbinfo->next_ino += SHMEM_INO_BATCH;
 384                         raw_spin_unlock(&sbinfo->stat_lock);
 385                         if (unlikely(is_zero_ino(ino)))
 386                                 ino++;
 387                 }
 388                 *inop = ino;
 389                 *next_ino = ++ino;
 390                 put_cpu();
 391         }
 392
 393         return 0;
 394 }
 395
 396 static void shmem_free_inode(struct super_block *sb, size_t freed_ispace)
 397 {
 398         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 399         if (sbinfo->max_inodes) {
 400                 raw_spin_lock(&sbinfo->stat_lock);
 401                 sbinfo->free_ispace += BOGO_INODE_SIZE + freed_ispace;
 402                 raw_spin_unlock(&sbinfo->stat_lock);
 403         }
 404 }
 405
 406 /**
 407  * shmem_recalc_inode - recalculate the block usage of an inode
 408  * @inode: inode to recalc
 409  * @alloced: the change in number of pages allocated to inode
 410  * @swapped: the change in number of pages swapped from inode
 411  *
 412  * We have to calculate the free blocks since the mm can drop
 413  * undirtied hole pages behind our back.
 414  *
 415  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
 416  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
 417  */
 418 static void shmem_recalc_inode(struct inode *inode, long alloced, long swapped)
 419 {
 420         struct shmem_inode_info *info = SHMEM_I(inode);
 421         long freed;
 422
 423         spin_lock(&info->lock);
 424         info->alloced += alloced;
 425         info->swapped += swapped;
 426         freed = info->alloced - info->swapped -
 427                 READ_ONCE(inode->i_mapping->nrpages);
 428         /*
 429          * Special case: whereas normally shmem_recalc_inode() is called
 430          * after i_mapping->nrpages has already been adjusted (up or down),
 431          * shmem_writepage() has to raise swapped before nrpages is lowered -
 432          * to stop a racing shmem_recalc_inode() from thinking that a page has
 433          * been freed.  Compensate here, to avoid the need for a followup call.
 434          */
 435         if (swapped > 0)
 436                 freed += swapped;
 437         if (freed > 0)
 438                 info->alloced -= freed;
 439         spin_unlock(&info->lock);
 440
 441         /* The quota case may block */
 442         if (freed > 0)
 443                 shmem_inode_unacct_blocks(inode, freed);
 444 }
 445
 446 bool shmem_charge(struct inode *inode, long pages)
 447 {
 448         struct address_space *mapping = inode->i_mapping;
 449
 450         if (shmem_inode_acct_blocks(inode, pages))
 451                 return false;
 452
 453         /* nrpages adjustment first, then shmem_recalc_inode() when balanced */
 454         xa_lock_irq(&mapping->i_pages);
 455         mapping->nrpages += pages;
 456         xa_unlock_irq(&mapping->i_pages);
 457
 458         shmem_recalc_inode(inode, pages, 0);
 459         return true;
 460 }
 461
 462 void shmem_uncharge(struct inode *inode, long pages)
 463 {
 464         /* pages argument is currently unused: keep it to help debugging */
 465         /* nrpages adjustment done by __filemap_remove_folio() or caller */
 466
 467         shmem_recalc_inode(inode, 0, 0);
 468 }
 469
 470 /*
 471  * Replace item expected in xarray by a new item, while holding xa_lock.
 472  */
 473 static int shmem_replace_entry(struct address_space *mapping,
 474                         pgoff_t index, void *expected, void *replacement)
 475 {
 476         XA_STATE(xas, &mapping->i_pages, index);
 477         void *item;
 478
 479         VM_BUG_ON(!expected);
 480         VM_BUG_ON(!replacement);
 481         item = xas_load(&xas);
 482         if (item != expected)
 483                 return -ENOENT;
 484         xas_store(&xas, replacement);
 485         return 0;
 486 }
 487
 488 /*
 489  * Sometimes, before we decide whether to proceed or to fail, we must check
 490  * that an entry was not already brought back from swap by a racing thread.
 491  *
 492  * Checking page is not enough: by the time a SwapCache page is locked, it
 493  * might be reused, and again be SwapCache, using the same swap as before.
 494  */
 495 static bool shmem_confirm_swap(struct address_space *mapping,
 496                                pgoff_t index, swp_entry_t swap)
 497 {
 498         return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
 499 }
 500
 501 /*
 502  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
 503  *
 504  * SHMEM_HUGE_NEVER:
 505  *      disables huge pages for the mount;
 506  * SHMEM_HUGE_ALWAYS:
 507  *      enables huge pages for the mount;
 508  * SHMEM_HUGE_WITHIN_SIZE:
 509  *      only allocate huge pages if the page will be fully within i_size,
 510  *      also respect fadvise()/madvise() hints;
 511  * SHMEM_HUGE_ADVISE:
 512  *      only allocate huge pages if requested with fadvise()/madvise();
 513  */
 514
 515 #define SHMEM_HUGE_NEVER        0
 516 #define SHMEM_HUGE_ALWAYS       1
 517 #define SHMEM_HUGE_WITHIN_SIZE  2
 518 #define SHMEM_HUGE_ADVISE       3
 519
 520 /*
 521  * Special values.
 522  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
 523  *
 524  * SHMEM_HUGE_DENY:
 525  *      disables huge on shm_mnt and all mounts, for emergency use;
 526  * SHMEM_HUGE_FORCE:
 527  *      enables huge on shm_mnt and all mounts, w/o needing option, for testing;
 528  *
 529  */
 530 #define SHMEM_HUGE_DENY         (-1)
 531 #define SHMEM_HUGE_FORCE        (-2)
 532
 533 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 534 /* ifdef here to avoid bloating shmem.o when not necessary */
 535
 536 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER;
 537
 538 bool shmem_is_huge(struct inode *inode, pgoff_t index, bool shmem_huge_force,
 539                    struct mm_struct *mm, unsigned long vm_flags)
 540 {
 541         loff_t i_size;
 542
 543         if (!S_ISREG(inode->i_mode))
 544                 return false;
 545         if (mm && ((vm_flags & VM_NOHUGEPAGE) || test_bit(MMF_DISABLE_THP, &mm->flags)))
 546                 return false;
 547         if (shmem_huge == SHMEM_HUGE_DENY)
 548                 return false;
 549         if (shmem_huge_force || shmem_huge == SHMEM_HUGE_FORCE)
 550                 return true;
 551
 552         switch (SHMEM_SB(inode->i_sb)->huge) {
 553         case SHMEM_HUGE_ALWAYS:
 554                 return true;
 555         case SHMEM_HUGE_WITHIN_SIZE:
 556                 index = round_up(index + 1, HPAGE_PMD_NR);
 557                 i_size = round_up(i_size_read(inode), PAGE_SIZE);
 558                 if (i_size >> PAGE_SHIFT >= index)
 559                         return true;
 560                 fallthrough;
 561         case SHMEM_HUGE_ADVISE:
 562                 if (mm && (vm_flags & VM_HUGEPAGE))
 563                         return true;
 564                 fallthrough;
 565         default:
 566                 return false;
 567         }
 568 }
 569
 570 #if defined(CONFIG_SYSFS)
 571 static int shmem_parse_huge(const char *str)
 572 {
 573         if (!strcmp(str, "never"))
 574                 return SHMEM_HUGE_NEVER;
 575         if (!strcmp(str, "always"))
 576                 return SHMEM_HUGE_ALWAYS;
 577         if (!strcmp(str, "within_size"))
 578                 return SHMEM_HUGE_WITHIN_SIZE;
 579         if (!strcmp(str, "advise"))
 580                 return SHMEM_HUGE_ADVISE;
 581         if (!strcmp(str, "deny"))
 582                 return SHMEM_HUGE_DENY;
 583         if (!strcmp(str, "force"))
 584                 return SHMEM_HUGE_FORCE;
 585         return -EINVAL;
 586 }
 587 #endif
 588
 589 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
 590 static const char *shmem_format_huge(int huge)
 591 {
 592         switch (huge) {
 593         case SHMEM_HUGE_NEVER:
 594                 return "never";
 595         case SHMEM_HUGE_ALWAYS:
 596                 return "always";
 597         case SHMEM_HUGE_WITHIN_SIZE:
 598                 return "within_size";
 599         case SHMEM_HUGE_ADVISE:
 600                 return "advise";
 601         case SHMEM_HUGE_DENY:
 602                 return "deny";
 603         case SHMEM_HUGE_FORCE:
 604                 return "force";
 605         default:
 606                 VM_BUG_ON(1);
 607                 return "bad_val";
 608         }
 609 }
 610 #endif
 611
 612 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
 613                 struct shrink_control *sc, unsigned long nr_to_split)
 614 {
 615         LIST_HEAD(list), *pos, *next;
 616         LIST_HEAD(to_remove);
 617         struct inode *inode;
 618         struct shmem_inode_info *info;
 619         struct folio *folio;
 620         unsigned long batch = sc ? sc->nr_to_scan : 128;
 621         int split = 0;
 622
 623         if (list_empty(&sbinfo->shrinklist))
 624                 return SHRINK_STOP;
 625
 626         spin_lock(&sbinfo->shrinklist_lock);
 627         list_for_each_safe(pos, next, &sbinfo->shrinklist) {
 628                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 629
 630                 /* pin the inode */
 631                 inode = igrab(&info->vfs_inode);
 632
 633                 /* inode is about to be evicted */
 634                 if (!inode) {
 635                         list_del_init(&info->shrinklist);
 636                         goto next;
 637                 }
 638
 639                 /* Check if there's anything to gain */
 640                 if (round_up(inode->i_size, PAGE_SIZE) ==
 641                                 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
 642                         list_move(&info->shrinklist, &to_remove);
 643                         goto next;
 644                 }
 645
 646                 list_move(&info->shrinklist, &list);
 647 next:
 648                 sbinfo->shrinklist_len--;
 649                 if (!--batch)
 650                         break;
 651         }
 652         spin_unlock(&sbinfo->shrinklist_lock);
 653
 654         list_for_each_safe(pos, next, &to_remove) {
 655                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 656                 inode = &info->vfs_inode;
 657                 list_del_init(&info->shrinklist);
 658                 iput(inode);
 659         }
 660
 661         list_for_each_safe(pos, next, &list) {
 662                 int ret;
 663                 pgoff_t index;
 664
 665                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
 666                 inode = &info->vfs_inode;
 667
 668                 if (nr_to_split && split >= nr_to_split)
 669                         goto move_back;
 670
 671                 index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT;
 672                 folio = filemap_get_folio(inode->i_mapping, index);
 673                 if (IS_ERR(folio))
 674                         goto drop;
 675
 676                 /* No huge page at the end of the file: nothing to split */
 677                 if (!folio_test_large(folio)) {
 678                         folio_put(folio);
 679                         goto drop;
 680                 }
 681
 682                 /*
 683                  * Move the inode on the list back to shrinklist if we failed
 684                  * to lock the page at this time.
 685                  *
 686                  * Waiting for the lock may lead to deadlock in the
 687                  * reclaim path.
 688                  */
 689                 if (!folio_trylock(folio)) {
 690                         folio_put(folio);
 691                         goto move_back;
 692                 }
 693
 694                 ret = split_folio(folio);
 695                 folio_unlock(folio);
 696                 folio_put(folio);
 697
 698                 /* If split failed move the inode on the list back to shrinklist */
 699                 if (ret)
 700                         goto move_back;
 701
 702                 split++;
 703 drop:
 704                 list_del_init(&info->shrinklist);
 705                 goto put;
 706 move_back:
 707                 /*
 708                  * Make sure the inode is either on the global list or deleted
 709                  * from any local list before iput() since it could be deleted
 710                  * in another thread once we put the inode (then the local list
 711                  * is corrupted).
 712                  */
 713                 spin_lock(&sbinfo->shrinklist_lock);
 714                 list_move(&info->shrinklist, &sbinfo->shrinklist);
 715                 sbinfo->shrinklist_len++;
 716                 spin_unlock(&sbinfo->shrinklist_lock);
 717 put:
 718                 iput(inode);
 719         }
 720
 721         return split;
 722 }
 723
 724 static long shmem_unused_huge_scan(struct super_block *sb,
 725                 struct shrink_control *sc)
 726 {
 727         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 728
 729         if (!READ_ONCE(sbinfo->shrinklist_len))
 730                 return SHRINK_STOP;
 731
 732         return shmem_unused_huge_shrink(sbinfo, sc, 0);
 733 }
 734
 735 static long shmem_unused_huge_count(struct super_block *sb,
 736                 struct shrink_control *sc)
 737 {
 738         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 739         return READ_ONCE(sbinfo->shrinklist_len);
 740 }
 741 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
 742
 743 #define shmem_huge SHMEM_HUGE_DENY
 744
 745 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
 746                 struct shrink_control *sc, unsigned long nr_to_split)
 747 {
 748         return 0;
 749 }
 750 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 751
 752 /*
 753  * Somewhat like filemap_add_folio, but error if expected item has gone.
 754  */
 755 static int shmem_add_to_page_cache(struct folio *folio,
 756                                    struct address_space *mapping,
 757                                    pgoff_t index, void *expected, gfp_t gfp)
 758 {
 759         XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio));
 760         long nr = folio_nr_pages(folio);
 761
 762         VM_BUG_ON_FOLIO(index != round_down(index, nr), folio);
 763         VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
 764         VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
 765         VM_BUG_ON(expected && folio_test_large(folio));
 766
 767         folio_ref_add(folio, nr);
 768         folio->mapping = mapping;
 769         folio->index = index;
 770
 771         gfp &= GFP_RECLAIM_MASK;
 772         folio_throttle_swaprate(folio, gfp);
 773
 774         do {
 775                 xas_lock_irq(&xas);
 776                 if (expected != xas_find_conflict(&xas)) {
 777                         xas_set_err(&xas, -EEXIST);
 778                         goto unlock;
 779                 }
 780                 if (expected && xas_find_conflict(&xas)) {
 781                         xas_set_err(&xas, -EEXIST);
 782                         goto unlock;
 783                 }
 784                 xas_store(&xas, folio);
 785                 if (xas_error(&xas))
 786                         goto unlock;
 787                 if (folio_test_pmd_mappable(folio))
 788                         __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr);
 789                 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
 790                 __lruvec_stat_mod_folio(folio, NR_SHMEM, nr);
 791                 mapping->nrpages += nr;
 792 unlock:
 793                 xas_unlock_irq(&xas);
 794         } while (xas_nomem(&xas, gfp));
 795
 796         if (xas_error(&xas)) {
 797                 folio->mapping = NULL;
 798                 folio_ref_sub(folio, nr);
 799                 return xas_error(&xas);
 800         }
 801
 802         return 0;
 803 }
 804
 805 /*
 806  * Somewhat like filemap_remove_folio, but substitutes swap for @folio.
 807  */
 808 static void shmem_delete_from_page_cache(struct folio *folio, void *radswap)
 809 {
 810         struct address_space *mapping = folio->mapping;
 811         long nr = folio_nr_pages(folio);
 812         int error;
 813
 814         xa_lock_irq(&mapping->i_pages);
 815         error = shmem_replace_entry(mapping, folio->index, folio, radswap);
 816         folio->mapping = NULL;
 817         mapping->nrpages -= nr;
 818         __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
 819         __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr);
 820         xa_unlock_irq(&mapping->i_pages);
 821         folio_put(folio);
 822         BUG_ON(error);
 823 }
 824
 825 /*
 826  * Remove swap entry from page cache, free the swap and its page cache.
 827  */
 828 static int shmem_free_swap(struct address_space *mapping,
 829                            pgoff_t index, void *radswap)
 830 {
 831         void *old;
 832
 833         old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
 834         if (old != radswap)
 835                 return -ENOENT;
 836         free_swap_and_cache(radix_to_swp_entry(radswap));
 837         return 0;
 838 }
 839
 840 /*
 841  * Determine (in bytes) how many of the shmem object's pages mapped by the
 842  * given offsets are swapped out.
 843  *
 844  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
 845  * as long as the inode doesn't go away and racy results are not a problem.
 846  */
 847 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
 848                                                 pgoff_t start, pgoff_t end)
 849 {
 850         XA_STATE(xas, &mapping->i_pages, start);
 851         struct page *page;
 852         unsigned long swapped = 0;
 853         unsigned long max = end - 1;
 854
 855         rcu_read_lock();
 856         xas_for_each(&xas, page, max) {
 857                 if (xas_retry(&xas, page))
 858                         continue;
 859                 if (xa_is_value(page))
 860                         swapped++;
 861                 if (xas.xa_index == max)
 862                         break;
 863                 if (need_resched()) {
 864                         xas_pause(&xas);
 865                         cond_resched_rcu();
 866                 }
 867         }
 868         rcu_read_unlock();
 869
 870         return swapped << PAGE_SHIFT;
 871 }
 872
 873 /*
 874  * Determine (in bytes) how many of the shmem object's pages mapped by the
 875  * given vma is swapped out.
 876  *
 877  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
 878  * as long as the inode doesn't go away and racy results are not a problem.
 879  */
 880 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
 881 {
 882         struct inode *inode = file_inode(vma->vm_file);
 883         struct shmem_inode_info *info = SHMEM_I(inode);
 884         struct address_space *mapping = inode->i_mapping;
 885         unsigned long swapped;
 886
 887         /* Be careful as we don't hold info->lock */
 888         swapped = READ_ONCE(info->swapped);
 889
 890         /*
 891          * The easier cases are when the shmem object has nothing in swap, or
 892          * the vma maps it whole. Then we can simply use the stats that we
 893          * already track.
 894          */
 895         if (!swapped)
 896                 return 0;
 897
 898         if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
 899                 return swapped << PAGE_SHIFT;
 900
 901         /* Here comes the more involved part */
 902         return shmem_partial_swap_usage(mapping, vma->vm_pgoff,
 903                                         vma->vm_pgoff + vma_pages(vma));
 904 }
 905
 906 /*
 907  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
 908  */
 909 void shmem_unlock_mapping(struct address_space *mapping)
 910 {
 911         struct folio_batch fbatch;
 912         pgoff_t index = 0;
 913
 914         folio_batch_init(&fbatch);
 915         /*
 916          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
 917          */
 918         while (!mapping_unevictable(mapping) &&
 919                filemap_get_folios(mapping, &index, ~0UL, &fbatch)) {
 920                 check_move_unevictable_folios(&fbatch);
 921                 folio_batch_release(&fbatch);
 922                 cond_resched();
 923         }
 924 }
 925
 926 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
 927 {
 928         struct folio *folio;
 929
 930         /*
 931          * At first avoid shmem_get_folio(,,,SGP_READ): that fails
 932          * beyond i_size, and reports fallocated folios as holes.
 933          */
 934         folio = filemap_get_entry(inode->i_mapping, index);
 935         if (!folio)
 936                 return folio;
 937         if (!xa_is_value(folio)) {
 938                 folio_lock(folio);
 939                 if (folio->mapping == inode->i_mapping)
 940                         return folio;
 941                 /* The folio has been swapped out */
 942                 folio_unlock(folio);
 943                 folio_put(folio);
 944         }
 945         /*
 946          * But read a folio back from swap if any of it is within i_size
 947          * (although in some cases this is just a waste of time).
 948          */
 949         folio = NULL;
 950         shmem_get_folio(inode, index, &folio, SGP_READ);
 951         return folio;
 952 }
 953
 954 /*
 955  * Remove range of pages and swap entries from page cache, and free them.
 956  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
 957  */
 958 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
 959                                                                  bool unfalloc)
 960 {
 961         struct address_space *mapping = inode->i_mapping;
 962         struct shmem_inode_info *info = SHMEM_I(inode);
 963         pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
 964         pgoff_t end = (lend + 1) >> PAGE_SHIFT;
 965         struct folio_batch fbatch;
 966         pgoff_t indices[PAGEVEC_SIZE];
 967         struct folio *folio;
 968         bool same_folio;
 969         long nr_swaps_freed = 0;
 970         pgoff_t index;
 971         int i;
 972
 973         if (lend == -1)
 974                 end = -1;       /* unsigned, so actually very big */
 975
 976         if (info->fallocend > start && info->fallocend <= end && !unfalloc)
 977                 info->fallocend = start;
 978
 979         folio_batch_init(&fbatch);
 980         index = start;
 981         while (index < end && find_lock_entries(mapping, &index, end - 1,
 982                         &fbatch, indices)) {
 983                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
 984                         folio = fbatch.folios[i];
 985
 986                         if (xa_is_value(folio)) {
 987                                 if (unfalloc)
 988                                         continue;
 989                                 nr_swaps_freed += !shmem_free_swap(mapping,
 990                                                         indices[i], folio);
 991                                 continue;
 992                         }
 993
 994                         if (!unfalloc || !folio_test_uptodate(folio))
 995                                 truncate_inode_folio(mapping, folio);
 996                         folio_unlock(folio);
 997                 }
 998                 folio_batch_remove_exceptionals(&fbatch);
 999                 folio_batch_release(&fbatch);
1000                 cond_resched();
1001         }
1002
1003         /*
1004          * When undoing a failed fallocate, we want none of the partial folio
1005          * zeroing and splitting below, but shall want to truncate the whole
1006          * folio when !uptodate indicates that it was added by this fallocate,
1007          * even when [lstart, lend] covers only a part of the folio.
1008          */
1009         if (unfalloc)
1010                 goto whole_folios;
1011
1012         same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
1013         folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT);
1014         if (folio) {
1015                 same_folio = lend < folio_pos(folio) + folio_size(folio);
1016                 folio_mark_dirty(folio);
1017                 if (!truncate_inode_partial_folio(folio, lstart, lend)) {
1018                         start = folio_next_index(folio);
1019                         if (same_folio)
1020                                 end = folio->index;
1021                 }
1022                 folio_unlock(folio);
1023                 folio_put(folio);
1024                 folio = NULL;
1025         }
1026
1027         if (!same_folio)
1028                 folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT);
1029         if (folio) {
1030                 folio_mark_dirty(folio);
1031                 if (!truncate_inode_partial_folio(folio, lstart, lend))
1032                         end = folio->index;
1033                 folio_unlock(folio);
1034                 folio_put(folio);
1035         }
1036
1037 whole_folios:
1038
1039         index = start;
1040         while (index < end) {
1041                 cond_resched();
1042
1043                 if (!find_get_entries(mapping, &index, end - 1, &fbatch,
1044                                 indices)) {
1045                         /* If all gone or hole-punch or unfalloc, we're done */
1046                         if (index == start || end != -1)
1047                                 break;
1048                         /* But if truncating, restart to make sure all gone */
1049                         index = start;
1050                         continue;
1051                 }
1052                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
1053                         folio = fbatch.folios[i];
1054
1055                         if (xa_is_value(folio)) {
1056                                 if (unfalloc)
1057                                         continue;
1058                                 if (shmem_free_swap(mapping, indices[i], folio)) {
1059                                         /* Swap was replaced by page: retry */
1060                                         index = indices[i];
1061                                         break;
1062                                 }
1063                                 nr_swaps_freed++;
1064                                 continue;
1065                         }
1066
1067                         folio_lock(folio);
1068
1069                         if (!unfalloc || !folio_test_uptodate(folio)) {
1070                                 if (folio_mapping(folio) != mapping) {
1071                                         /* Page was replaced by swap: retry */
1072                                         folio_unlock(folio);
1073                                         index = indices[i];
1074                                         break;
1075                                 }
1076                                 VM_BUG_ON_FOLIO(folio_test_writeback(folio),
1077                                                 folio);
1078
1079                                 if (!folio_test_large(folio)) {
1080                                         truncate_inode_folio(mapping, folio);
1081                                 } else if (truncate_inode_partial_folio(folio, lstart, lend)) {
1082                                         /*
1083                                          * If we split a page, reset the loop so
1084                                          * that we pick up the new sub pages.
1085                                          * Otherwise the THP was entirely
1086                                          * dropped or the target range was
1087                                          * zeroed, so just continue the loop as
1088                                          * is.
1089                                          */
1090                                         if (!folio_test_large(folio)) {
1091                                                 folio_unlock(folio);
1092                                                 index = start;
1093                                                 break;
1094                                         }
1095                                 }
1096                         }
1097                         folio_unlock(folio);
1098                 }
1099                 folio_batch_remove_exceptionals(&fbatch);
1100                 folio_batch_release(&fbatch);
1101         }
1102
1103         shmem_recalc_inode(inode, 0, -nr_swaps_freed);
1104 }
1105
1106 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
1107 {
1108         shmem_undo_range(inode, lstart, lend, false);
1109         inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
1110         inode_inc_iversion(inode);
1111 }
1112 EXPORT_SYMBOL_GPL(shmem_truncate_range);
1113
1114 static int shmem_getattr(struct mnt_idmap *idmap,
1115                          const struct path *path, struct kstat *stat,
1116                          u32 request_mask, unsigned int query_flags)
1117 {
1118         struct inode *inode = path->dentry->d_inode;
1119         struct shmem_inode_info *info = SHMEM_I(inode);
1120
1121         if (info->alloced - info->swapped != inode->i_mapping->nrpages)
1122                 shmem_recalc_inode(inode, 0, 0);
1123
1124         if (info->fsflags & FS_APPEND_FL)
1125                 stat->attributes |= STATX_ATTR_APPEND;
1126         if (info->fsflags & FS_IMMUTABLE_FL)
1127                 stat->attributes |= STATX_ATTR_IMMUTABLE;
1128         if (info->fsflags & FS_NODUMP_FL)
1129                 stat->attributes |= STATX_ATTR_NODUMP;
1130         stat->attributes_mask |= (STATX_ATTR_APPEND |
1131                         STATX_ATTR_IMMUTABLE |
1132                         STATX_ATTR_NODUMP);
1133         generic_fillattr(idmap, request_mask, inode, stat);
1134
1135         if (shmem_is_huge(inode, 0, false, NULL, 0))
1136                 stat->blksize = HPAGE_PMD_SIZE;
1137
1138         if (request_mask & STATX_BTIME) {
1139                 stat->result_mask |= STATX_BTIME;
1140                 stat->btime.tv_sec = info->i_crtime.tv_sec;
1141                 stat->btime.tv_nsec = info->i_crtime.tv_nsec;
1142         }
1143
1144         return 0;
1145 }
1146
1147 static int shmem_setattr(struct mnt_idmap *idmap,
1148                          struct dentry *dentry, struct iattr *attr)
1149 {
1150         struct inode *inode = d_inode(dentry);
1151         struct shmem_inode_info *info = SHMEM_I(inode);
1152         int error;
1153         bool update_mtime = false;
1154         bool update_ctime = true;
1155
1156         error = setattr_prepare(idmap, dentry, attr);
1157         if (error)
1158                 return error;
1159
1160         if ((info->seals & F_SEAL_EXEC) && (attr->ia_valid & ATTR_MODE)) {
1161                 if ((inode->i_mode ^ attr->ia_mode) & 0111) {
1162                         return -EPERM;
1163                 }
1164         }
1165
1166         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1167                 loff_t oldsize = inode->i_size;
1168                 loff_t newsize = attr->ia_size;
1169
1170                 /* protected by i_rwsem */
1171                 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1172                     (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1173                         return -EPERM;
1174
1175                 if (newsize != oldsize) {
1176                         error = shmem_reacct_size(SHMEM_I(inode)->flags,
1177                                         oldsize, newsize);
1178                         if (error)
1179                                 return error;
1180                         i_size_write(inode, newsize);
1181                         update_mtime = true;
1182                 } else {
1183                         update_ctime = false;
1184                 }
1185                 if (newsize <= oldsize) {
1186                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
1187                         if (oldsize > holebegin)
1188                                 unmap_mapping_range(inode->i_mapping,
1189                                                         holebegin, 0, 1);
1190                         if (info->alloced)
1191                                 shmem_truncate_range(inode,
1192                                                         newsize, (loff_t)-1);
1193                         /* unmap again to remove racily COWed private pages */
1194                         if (oldsize > holebegin)
1195                                 unmap_mapping_range(inode->i_mapping,
1196                                                         holebegin, 0, 1);
1197                 }
1198         }
1199
1200         if (is_quota_modification(idmap, inode, attr)) {
1201                 error = dquot_initialize(inode);
1202                 if (error)
1203                         return error;
1204         }
1205
1206         /* Transfer quota accounting */
1207         if (i_uid_needs_update(idmap, attr, inode) ||
1208             i_gid_needs_update(idmap, attr, inode)) {
1209                 error = dquot_transfer(idmap, inode, attr);
1210                 if (error)
1211                         return error;
1212         }
1213
1214         setattr_copy(idmap, inode, attr);
1215         if (attr->ia_valid & ATTR_MODE)
1216                 error = posix_acl_chmod(idmap, dentry, inode->i_mode);
1217         if (!error && update_ctime) {
1218                 inode_set_ctime_current(inode);
1219                 if (update_mtime)
1220                         inode_set_mtime_to_ts(inode, inode_get_ctime(inode));
1221                 inode_inc_iversion(inode);
1222         }
1223         return error;
1224 }
1225
1226 static void shmem_evict_inode(struct inode *inode)
1227 {
1228         struct shmem_inode_info *info = SHMEM_I(inode);
1229         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1230         size_t freed = 0;
1231
1232         if (shmem_mapping(inode->i_mapping)) {
1233                 shmem_unacct_size(info->flags, inode->i_size);
1234                 inode->i_size = 0;
1235                 mapping_set_exiting(inode->i_mapping);
1236                 shmem_truncate_range(inode, 0, (loff_t)-1);
1237                 if (!list_empty(&info->shrinklist)) {
1238                         spin_lock(&sbinfo->shrinklist_lock);
1239                         if (!list_empty(&info->shrinklist)) {
1240                                 list_del_init(&info->shrinklist);
1241                                 sbinfo->shrinklist_len--;
1242                         }
1243                         spin_unlock(&sbinfo->shrinklist_lock);
1244                 }
1245                 while (!list_empty(&info->swaplist)) {
1246                         /* Wait while shmem_unuse() is scanning this inode... */
1247                         wait_var_event(&info->stop_eviction,
1248                                        !atomic_read(&info->stop_eviction));
1249                         mutex_lock(&shmem_swaplist_mutex);
1250                         /* ...but beware of the race if we peeked too early */
1251                         if (!atomic_read(&info->stop_eviction))
1252                                 list_del_init(&info->swaplist);
1253                         mutex_unlock(&shmem_swaplist_mutex);
1254                 }
1255         }
1256
1257         simple_xattrs_free(&info->xattrs, sbinfo->max_inodes ? &freed : NULL);
1258         shmem_free_inode(inode->i_sb, freed);
1259         WARN_ON(inode->i_blocks);
1260         clear_inode(inode);
1261 #ifdef CONFIG_TMPFS_QUOTA
1262         dquot_free_inode(inode);
1263         dquot_drop(inode);
1264 #endif
1265 }
1266
1267 static int shmem_find_swap_entries(struct address_space *mapping,
1268                                    pgoff_t start, struct folio_batch *fbatch,
1269                                    pgoff_t *indices, unsigned int type)
1270 {
1271         XA_STATE(xas, &mapping->i_pages, start);
1272         struct folio *folio;
1273         swp_entry_t entry;
1274
1275         rcu_read_lock();
1276         xas_for_each(&xas, folio, ULONG_MAX) {
1277                 if (xas_retry(&xas, folio))
1278                         continue;
1279
1280                 if (!xa_is_value(folio))
1281                         continue;
1282
1283                 entry = radix_to_swp_entry(folio);
1284                 /*
1285                  * swapin error entries can be found in the mapping. But they're
1286                  * deliberately ignored here as we've done everything we can do.
1287                  */
1288                 if (swp_type(entry) != type)
1289                         continue;
1290
1291                 indices[folio_batch_count(fbatch)] = xas.xa_index;
1292                 if (!folio_batch_add(fbatch, folio))
1293                         break;
1294
1295                 if (need_resched()) {
1296                         xas_pause(&xas);
1297                         cond_resched_rcu();
1298                 }
1299         }
1300         rcu_read_unlock();
1301
1302         return xas.xa_index;
1303 }
1304
1305 /*
1306  * Move the swapped pages for an inode to page cache. Returns the count
1307  * of pages swapped in, or the error in case of failure.
1308  */
1309 static int shmem_unuse_swap_entries(struct inode *inode,
1310                 struct folio_batch *fbatch, pgoff_t *indices)
1311 {
1312         int i = 0;
1313         int ret = 0;
1314         int error = 0;
1315         struct address_space *mapping = inode->i_mapping;
1316
1317         for (i = 0; i < folio_batch_count(fbatch); i++) {
1318                 struct folio *folio = fbatch->folios[i];
1319
1320                 if (!xa_is_value(folio))
1321                         continue;
1322                 error = shmem_swapin_folio(inode, indices[i], &folio, SGP_CACHE,
1323                                         mapping_gfp_mask(mapping), NULL, NULL);
1324                 if (error == 0) {
1325                         folio_unlock(folio);
1326                         folio_put(folio);
1327                         ret++;
1328                 }
1329                 if (error == -ENOMEM)
1330                         break;
1331                 error = 0;
1332         }
1333         return error ? error : ret;
1334 }
1335
1336 /*
1337  * If swap found in inode, free it and move page from swapcache to filecache.
1338  */
1339 static int shmem_unuse_inode(struct inode *inode, unsigned int type)
1340 {
1341         struct address_space *mapping = inode->i_mapping;
1342         pgoff_t start = 0;
1343         struct folio_batch fbatch;
1344         pgoff_t indices[PAGEVEC_SIZE];
1345         int ret = 0;
1346
1347         do {
1348                 folio_batch_init(&fbatch);
1349                 shmem_find_swap_entries(mapping, start, &fbatch, indices, type);
1350                 if (folio_batch_count(&fbatch) == 0) {
1351                         ret = 0;
1352                         break;
1353                 }
1354
1355                 ret = shmem_unuse_swap_entries(inode, &fbatch, indices);
1356                 if (ret < 0)
1357                         break;
1358
1359                 start = indices[folio_batch_count(&fbatch) - 1];
1360         } while (true);
1361
1362         return ret;
1363 }
1364
1365 /*
1366  * Read all the shared memory data that resides in the swap
1367  * device 'type' back into memory, so the swap device can be
1368  * unused.
1369  */
1370 int shmem_unuse(unsigned int type)
1371 {
1372         struct shmem_inode_info *info, *next;
1373         int error = 0;
1374
1375         if (list_empty(&shmem_swaplist))
1376                 return 0;
1377
1378         mutex_lock(&shmem_swaplist_mutex);
1379         list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1380                 if (!info->swapped) {
1381                         list_del_init(&info->swaplist);
1382                         continue;
1383                 }
1384                 /*
1385                  * Drop the swaplist mutex while searching the inode for swap;
1386                  * but before doing so, make sure shmem_evict_inode() will not
1387                  * remove placeholder inode from swaplist, nor let it be freed
1388                  * (igrab() would protect from unlink, but not from unmount).
1389                  */
1390                 atomic_inc(&info->stop_eviction);
1391                 mutex_unlock(&shmem_swaplist_mutex);
1392
1393                 error = shmem_unuse_inode(&info->vfs_inode, type);
1394                 cond_resched();
1395
1396                 mutex_lock(&shmem_swaplist_mutex);
1397                 next = list_next_entry(info, swaplist);
1398                 if (!info->swapped)
1399                         list_del_init(&info->swaplist);
1400                 if (atomic_dec_and_test(&info->stop_eviction))
1401                         wake_up_var(&info->stop_eviction);
1402                 if (error)
1403                         break;
1404         }
1405         mutex_unlock(&shmem_swaplist_mutex);
1406
1407         return error;
1408 }
1409
1410 /*
1411  * Move the page from the page cache to the swap cache.
1412  */
1413 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1414 {
1415         struct folio *folio = page_folio(page);
1416         struct address_space *mapping = folio->mapping;
1417         struct inode *inode = mapping->host;
1418         struct shmem_inode_info *info = SHMEM_I(inode);
1419         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1420         swp_entry_t swap;
1421         pgoff_t index;
1422
1423         /*
1424          * Our capabilities prevent regular writeback or sync from ever calling
1425          * shmem_writepage; but a stacking filesystem might use ->writepage of
1426          * its underlying filesystem, in which case tmpfs should write out to
1427          * swap only in response to memory pressure, and not for the writeback
1428          * threads or sync.
1429          */
1430         if (WARN_ON_ONCE(!wbc->for_reclaim))
1431                 goto redirty;
1432
1433         if (WARN_ON_ONCE((info->flags & VM_LOCKED) || sbinfo->noswap))
1434                 goto redirty;
1435
1436         if (!total_swap_pages)
1437                 goto redirty;
1438
1439         /*
1440          * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or
1441          * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages,
1442          * and its shmem_writeback() needs them to be split when swapping.
1443          */
1444         if (folio_test_large(folio)) {
1445                 /* Ensure the subpages are still dirty */
1446                 folio_test_set_dirty(folio);
1447                 if (split_huge_page(page) < 0)
1448                         goto redirty;
1449                 folio = page_folio(page);
1450                 folio_clear_dirty(folio);
1451         }
1452
1453         index = folio->index;
1454
1455         /*
1456          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1457          * value into swapfile.c, the only way we can correctly account for a
1458          * fallocated folio arriving here is now to initialize it and write it.
1459          *
1460          * That's okay for a folio already fallocated earlier, but if we have
1461          * not yet completed the fallocation, then (a) we want to keep track
1462          * of this folio in case we have to undo it, and (b) it may not be a
1463          * good idea to continue anyway, once we're pushing into swap.  So
1464          * reactivate the folio, and let shmem_fallocate() quit when too many.
1465          */
1466         if (!folio_test_uptodate(folio)) {
1467                 if (inode->i_private) {
1468                         struct shmem_falloc *shmem_falloc;
1469                         spin_lock(&inode->i_lock);
1470                         shmem_falloc = inode->i_private;
1471                         if (shmem_falloc &&
1472                             !shmem_falloc->waitq &&
1473                             index >= shmem_falloc->start &&
1474                             index < shmem_falloc->next)
1475                                 shmem_falloc->nr_unswapped++;
1476                         else
1477                                 shmem_falloc = NULL;
1478                         spin_unlock(&inode->i_lock);
1479                         if (shmem_falloc)
1480                                 goto redirty;
1481                 }
1482                 folio_zero_range(folio, 0, folio_size(folio));
1483                 flush_dcache_folio(folio);
1484                 folio_mark_uptodate(folio);
1485         }
1486
1487         swap = folio_alloc_swap(folio);
1488         if (!swap.val)
1489                 goto redirty;
1490
1491         /*
1492          * Add inode to shmem_unuse()'s list of swapped-out inodes,
1493          * if it's not already there.  Do it now before the folio is
1494          * moved to swap cache, when its pagelock no longer protects
1495          * the inode from eviction.  But don't unlock the mutex until
1496          * we've incremented swapped, because shmem_unuse_inode() will
1497          * prune a !swapped inode from the swaplist under this mutex.
1498          */
1499         mutex_lock(&shmem_swaplist_mutex);
1500         if (list_empty(&info->swaplist))
1501                 list_add(&info->swaplist, &shmem_swaplist);
1502
1503         if (add_to_swap_cache(folio, swap,
1504                         __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
1505                         NULL) == 0) {
1506                 shmem_recalc_inode(inode, 0, 1);
1507                 swap_shmem_alloc(swap);
1508                 shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap));
1509
1510                 mutex_unlock(&shmem_swaplist_mutex);
1511                 BUG_ON(folio_mapped(folio));
1512                 return swap_writepage(&folio->page, wbc);
1513         }
1514
1515         mutex_unlock(&shmem_swaplist_mutex);
1516         put_swap_folio(folio, swap);
1517 redirty:
1518         folio_mark_dirty(folio);
1519         if (wbc->for_reclaim)
1520                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with folio locked */
1521         folio_unlock(folio);
1522         return 0;
1523 }
1524
1525 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1526 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1527 {
1528         char buffer[64];
1529
1530         if (!mpol || mpol->mode == MPOL_DEFAULT)
1531                 return;         /* show nothing */
1532
1533         mpol_to_str(buffer, sizeof(buffer), mpol);
1534
1535         seq_printf(seq, ",mpol=%s", buffer);
1536 }
1537
1538 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1539 {
1540         struct mempolicy *mpol = NULL;
1541         if (sbinfo->mpol) {
1542                 raw_spin_lock(&sbinfo->stat_lock);      /* prevent replace/use races */
1543                 mpol = sbinfo->mpol;
1544                 mpol_get(mpol);
1545                 raw_spin_unlock(&sbinfo->stat_lock);
1546         }
1547         return mpol;
1548 }
1549 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1550 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1551 {
1552 }
1553 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1554 {
1555         return NULL;
1556 }
1557 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1558
1559 static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info,
1560                         pgoff_t index, unsigned int order, pgoff_t *ilx);
1561
1562 static struct folio *shmem_swapin_cluster(swp_entry_t swap, gfp_t gfp,
1563                         struct shmem_inode_info *info, pgoff_t index)
1564 {
1565         struct mempolicy *mpol;
1566         pgoff_t ilx;
1567         struct folio *folio;
1568
1569         mpol = shmem_get_pgoff_policy(info, index, 0, &ilx);
1570         folio = swap_cluster_readahead(swap, gfp, mpol, ilx);
1571         mpol_cond_put(mpol);
1572
1573         return folio;
1574 }
1575
1576 /*
1577  * Make sure huge_gfp is always more limited than limit_gfp.
1578  * Some of the flags set permissions, while others set limitations.
1579  */
1580 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp)
1581 {
1582         gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM;
1583         gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY;
1584         gfp_t zoneflags = limit_gfp & GFP_ZONEMASK;
1585         gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK);
1586
1587         /* Allow allocations only from the originally specified zones. */
1588         result |= zoneflags;
1589
1590         /*
1591          * Minimize the result gfp by taking the union with the deny flags,
1592          * and the intersection of the allow flags.
1593          */
1594         result |= (limit_gfp & denyflags);
1595         result |= (huge_gfp & limit_gfp) & allowflags;
1596
1597         return result;
1598 }
1599
1600 static struct folio *shmem_alloc_hugefolio(gfp_t gfp,
1601                 struct shmem_inode_info *info, pgoff_t index)
1602 {
1603         struct mempolicy *mpol;
1604         pgoff_t ilx;
1605         struct page *page;
1606
1607         mpol = shmem_get_pgoff_policy(info, index, HPAGE_PMD_ORDER, &ilx);
1608         page = alloc_pages_mpol(gfp, HPAGE_PMD_ORDER, mpol, ilx, numa_node_id());
1609         mpol_cond_put(mpol);
1610
1611         return page_rmappable_folio(page);
1612 }
1613
1614 static struct folio *shmem_alloc_folio(gfp_t gfp,
1615                 struct shmem_inode_info *info, pgoff_t index)
1616 {
1617         struct mempolicy *mpol;
1618         pgoff_t ilx;
1619         struct page *page;
1620
1621         mpol = shmem_get_pgoff_policy(info, index, 0, &ilx);
1622         page = alloc_pages_mpol(gfp, 0, mpol, ilx, numa_node_id());
1623         mpol_cond_put(mpol);
1624
1625         return (struct folio *)page;
1626 }
1627
1628 static struct folio *shmem_alloc_and_add_folio(gfp_t gfp,
1629                 struct inode *inode, pgoff_t index,
1630                 struct mm_struct *fault_mm, bool huge)
1631 {
1632         struct address_space *mapping = inode->i_mapping;
1633         struct shmem_inode_info *info = SHMEM_I(inode);
1634         struct folio *folio;
1635         long pages;
1636         int error;
1637
1638         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1639                 huge = false;
1640
1641         if (huge) {
1642                 pages = HPAGE_PMD_NR;
1643                 index = round_down(index, HPAGE_PMD_NR);
1644
1645                 /*
1646                  * Check for conflict before waiting on a huge allocation.
1647                  * Conflict might be that a huge page has just been allocated
1648                  * and added to page cache by a racing thread, or that there
1649                  * is already at least one small page in the huge extent.
1650                  * Be careful to retry when appropriate, but not forever!
1651                  * Elsewhere -EEXIST would be the right code, but not here.
1652                  */
1653                 if (xa_find(&mapping->i_pages, &index,
1654                                 index + HPAGE_PMD_NR - 1, XA_PRESENT))
1655                         return ERR_PTR(-E2BIG);
1656
1657                 folio = shmem_alloc_hugefolio(gfp, info, index);
1658                 if (!folio)
1659                         count_vm_event(THP_FILE_FALLBACK);
1660         } else {
1661                 pages = 1;
1662                 folio = shmem_alloc_folio(gfp, info, index);
1663         }
1664         if (!folio)
1665                 return ERR_PTR(-ENOMEM);
1666
1667         __folio_set_locked(folio);
1668         __folio_set_swapbacked(folio);
1669
1670         gfp &= GFP_RECLAIM_MASK;
1671         error = mem_cgroup_charge(folio, fault_mm, gfp);
1672         if (error) {
1673                 if (xa_find(&mapping->i_pages, &index,
1674                                 index + pages - 1, XA_PRESENT)) {
1675                         error = -EEXIST;
1676                 } else if (huge) {
1677                         count_vm_event(THP_FILE_FALLBACK);
1678                         count_vm_event(THP_FILE_FALLBACK_CHARGE);
1679                 }
1680                 goto unlock;
1681         }
1682
1683         error = shmem_add_to_page_cache(folio, mapping, index, NULL, gfp);
1684         if (error)
1685                 goto unlock;
1686
1687         error = shmem_inode_acct_blocks(inode, pages);
1688         if (error) {
1689                 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1690                 long freed;
1691                 /*
1692                  * Try to reclaim some space by splitting a few
1693                  * large folios beyond i_size on the filesystem.
1694                  */
1695                 shmem_unused_huge_shrink(sbinfo, NULL, 2);
1696                 /*
1697                  * And do a shmem_recalc_inode() to account for freed pages:
1698                  * except our folio is there in cache, so not quite balanced.
1699                  */
1700                 spin_lock(&info->lock);
1701                 freed = pages + info->alloced - info->swapped -
1702                         READ_ONCE(mapping->nrpages);
1703                 if (freed > 0)
1704                         info->alloced -= freed;
1705                 spin_unlock(&info->lock);
1706                 if (freed > 0)
1707                         shmem_inode_unacct_blocks(inode, freed);
1708                 error = shmem_inode_acct_blocks(inode, pages);
1709                 if (error) {
1710                         filemap_remove_folio(folio);
1711                         goto unlock;
1712                 }
1713         }
1714
1715         shmem_recalc_inode(inode, pages, 0);
1716         folio_add_lru(folio);
1717         return folio;
1718
1719 unlock:
1720         folio_unlock(folio);
1721         folio_put(folio);
1722         return ERR_PTR(error);
1723 }
1724
1725 /*
1726  * When a page is moved from swapcache to shmem filecache (either by the
1727  * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of
1728  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1729  * ignorance of the mapping it belongs to.  If that mapping has special
1730  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1731  * we may need to copy to a suitable page before moving to filecache.
1732  *
1733  * In a future release, this may well be extended to respect cpuset and
1734  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1735  * but for now it is a simple matter of zone.
1736  */
1737 static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp)
1738 {
1739         return folio_zonenum(folio) > gfp_zone(gfp);
1740 }
1741
1742 static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
1743                                 struct shmem_inode_info *info, pgoff_t index)
1744 {
1745         struct folio *old, *new;
1746         struct address_space *swap_mapping;
1747         swp_entry_t entry;
1748         pgoff_t swap_index;
1749         int error;
1750
1751         old = *foliop;
1752         entry = old->swap;
1753         swap_index = swp_offset(entry);
1754         swap_mapping = swap_address_space(entry);
1755
1756         /*
1757          * We have arrived here because our zones are constrained, so don't
1758          * limit chance of success by further cpuset and node constraints.
1759          */
1760         gfp &= ~GFP_CONSTRAINT_MASK;
1761         VM_BUG_ON_FOLIO(folio_test_large(old), old);
1762         new = shmem_alloc_folio(gfp, info, index);
1763         if (!new)
1764                 return -ENOMEM;
1765
1766         folio_get(new);
1767         folio_copy(new, old);
1768         flush_dcache_folio(new);
1769
1770         __folio_set_locked(new);
1771         __folio_set_swapbacked(new);
1772         folio_mark_uptodate(new);
1773         new->swap = entry;
1774         folio_set_swapcache(new);
1775
1776         /*
1777          * Our caller will very soon move newpage out of swapcache, but it's
1778          * a nice clean interface for us to replace oldpage by newpage there.
1779          */
1780         xa_lock_irq(&swap_mapping->i_pages);
1781         error = shmem_replace_entry(swap_mapping, swap_index, old, new);
1782         if (!error) {
1783                 mem_cgroup_migrate(old, new);
1784                 __lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1);
1785                 __lruvec_stat_mod_folio(new, NR_SHMEM, 1);
1786                 __lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1);
1787                 __lruvec_stat_mod_folio(old, NR_SHMEM, -1);
1788         }
1789         xa_unlock_irq(&swap_mapping->i_pages);
1790
1791         if (unlikely(error)) {
1792                 /*
1793                  * Is this possible?  I think not, now that our callers check
1794                  * both PageSwapCache and page_private after getting page lock;
1795                  * but be defensive.  Reverse old to newpage for clear and free.
1796                  */
1797                 old = new;
1798         } else {
1799                 folio_add_lru(new);
1800                 *foliop = new;
1801         }
1802
1803         folio_clear_swapcache(old);
1804         old->private = NULL;
1805
1806         folio_unlock(old);
1807         folio_put_refs(old, 2);
1808         return error;
1809 }
1810
1811 static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index,
1812                                          struct folio *folio, swp_entry_t swap)
1813 {
1814         struct address_space *mapping = inode->i_mapping;
1815         swp_entry_t swapin_error;
1816         void *old;
1817
1818         swapin_error = make_poisoned_swp_entry();
1819         old = xa_cmpxchg_irq(&mapping->i_pages, index,
1820                              swp_to_radix_entry(swap),
1821                              swp_to_radix_entry(swapin_error), 0);
1822         if (old != swp_to_radix_entry(swap))
1823                 return;
1824
1825         folio_wait_writeback(folio);
1826         delete_from_swap_cache(folio);
1827         /*
1828          * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks
1829          * won't be 0 when inode is released and thus trigger WARN_ON(i_blocks)
1830          * in shmem_evict_inode().
1831          */
1832         shmem_recalc_inode(inode, -1, -1);
1833         swap_free(swap);
1834 }
1835
1836 /*
1837  * Swap in the folio pointed to by *foliop.
1838  * Caller has to make sure that *foliop contains a valid swapped folio.
1839  * Returns 0 and the folio in foliop if success. On failure, returns the
1840  * error code and NULL in *foliop.
1841  */
1842 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
1843                              struct folio **foliop, enum sgp_type sgp,
1844                              gfp_t gfp, struct mm_struct *fault_mm,
1845                              vm_fault_t *fault_type)
1846 {
1847         struct address_space *mapping = inode->i_mapping;
1848         struct shmem_inode_info *info = SHMEM_I(inode);
1849         struct swap_info_struct *si;
1850         struct folio *folio = NULL;
1851         swp_entry_t swap;
1852         int error;
1853
1854         VM_BUG_ON(!*foliop || !xa_is_value(*foliop));
1855         swap = radix_to_swp_entry(*foliop);
1856         *foliop = NULL;
1857
1858         if (is_poisoned_swp_entry(swap))
1859                 return -EIO;
1860
1861         si = get_swap_device(swap);
1862         if (!si) {
1863                 if (!shmem_confirm_swap(mapping, index, swap))
1864                         return -EEXIST;
1865                 else
1866                         return -EINVAL;
1867         }
1868
1869         /* Look it up and read it in.. */
1870         folio = swap_cache_get_folio(swap, NULL, 0);
1871         if (!folio) {
1872                 /* Or update major stats only when swapin succeeds?? */
1873                 if (fault_type) {
1874                         *fault_type |= VM_FAULT_MAJOR;
1875                         count_vm_event(PGMAJFAULT);
1876                         count_memcg_event_mm(fault_mm, PGMAJFAULT);
1877                 }
1878                 /* Here we actually start the io */
1879                 folio = shmem_swapin_cluster(swap, gfp, info, index);
1880                 if (!folio) {
1881                         error = -ENOMEM;
1882                         goto failed;
1883                 }
1884         }
1885
1886         /* We have to do this with folio locked to prevent races */
1887         folio_lock(folio);
1888         if (!folio_test_swapcache(folio) ||
1889             folio->swap.val != swap.val ||
1890             !shmem_confirm_swap(mapping, index, swap)) {
1891                 error = -EEXIST;
1892                 goto unlock;
1893         }
1894         if (!folio_test_uptodate(folio)) {
1895                 error = -EIO;
1896                 goto failed;
1897         }
1898         folio_wait_writeback(folio);
1899
1900         /*
1901          * Some architectures may have to restore extra metadata to the
1902          * folio after reading from swap.
1903          */
1904         arch_swap_restore(swap, folio);
1905
1906         if (shmem_should_replace_folio(folio, gfp)) {
1907                 error = shmem_replace_folio(&folio, gfp, info, index);
1908                 if (error)
1909                         goto failed;
1910         }
1911
1912         error = shmem_add_to_page_cache(folio, mapping, index,
1913                                         swp_to_radix_entry(swap), gfp);
1914         if (error)
1915                 goto failed;
1916
1917         shmem_recalc_inode(inode, 0, -1);
1918
1919         if (sgp == SGP_WRITE)
1920                 folio_mark_accessed(folio);
1921
1922         delete_from_swap_cache(folio);
1923         folio_mark_dirty(folio);
1924         swap_free(swap);
1925         put_swap_device(si);
1926
1927         *foliop = folio;
1928         return 0;
1929 failed:
1930         if (!shmem_confirm_swap(mapping, index, swap))
1931                 error = -EEXIST;
1932         if (error == -EIO)
1933                 shmem_set_folio_swapin_error(inode, index, folio, swap);
1934 unlock:
1935         if (folio) {
1936                 folio_unlock(folio);
1937                 folio_put(folio);
1938         }
1939         put_swap_device(si);
1940
1941         return error;
1942 }
1943
1944 /*
1945  * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate
1946  *
1947  * If we allocate a new one we do not mark it dirty. That's up to the
1948  * vm. If we swap it in we mark it dirty since we also free the swap
1949  * entry since a page cannot live in both the swap and page cache.
1950  *
1951  * vmf and fault_type are only supplied by shmem_fault: otherwise they are NULL.
1952  */
1953 static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
1954                 struct folio **foliop, enum sgp_type sgp, gfp_t gfp,
1955                 struct vm_fault *vmf, vm_fault_t *fault_type)
1956 {
1957         struct vm_area_struct *vma = vmf ? vmf->vma : NULL;
1958         struct mm_struct *fault_mm;
1959         struct folio *folio;
1960         int error;
1961         bool alloced;
1962
1963         if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1964                 return -EFBIG;
1965 repeat:
1966         if (sgp <= SGP_CACHE &&
1967             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode))
1968                 return -EINVAL;
1969
1970         alloced = false;
1971         fault_mm = vma ? vma->vm_mm : NULL;
1972
1973         folio = filemap_get_entry(inode->i_mapping, index);
1974         if (folio && vma && userfaultfd_minor(vma)) {
1975                 if (!xa_is_value(folio))
1976                         folio_put(folio);
1977                 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR);
1978                 return 0;
1979         }
1980
1981         if (xa_is_value(folio)) {
1982                 error = shmem_swapin_folio(inode, index, &folio,
1983                                            sgp, gfp, fault_mm, fault_type);
1984                 if (error == -EEXIST)
1985                         goto repeat;
1986
1987                 *foliop = folio;
1988                 return error;
1989         }
1990
1991         if (folio) {
1992                 folio_lock(folio);
1993
1994                 /* Has the folio been truncated or swapped out? */
1995                 if (unlikely(folio->mapping != inode->i_mapping)) {
1996                         folio_unlock(folio);
1997                         folio_put(folio);
1998                         goto repeat;
1999                 }
2000                 if (sgp == SGP_WRITE)
2001                         folio_mark_accessed(folio);
2002                 if (folio_test_uptodate(folio))
2003                         goto out;
2004                 /* fallocated folio */
2005                 if (sgp != SGP_READ)
2006                         goto clear;
2007                 folio_unlock(folio);
2008                 folio_put(folio);
2009         }
2010
2011         /*
2012          * SGP_READ: succeed on hole, with NULL folio, letting caller zero.
2013          * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail.
2014          */
2015         *foliop = NULL;
2016         if (sgp == SGP_READ)
2017                 return 0;
2018         if (sgp == SGP_NOALLOC)
2019                 return -ENOENT;
2020
2021         /*
2022          * Fast cache lookup and swap lookup did not find it: allocate.
2023          */
2024
2025         if (vma && userfaultfd_missing(vma)) {
2026                 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
2027                 return 0;
2028         }
2029
2030         if (shmem_is_huge(inode, index, false, fault_mm,
2031                           vma ? vma->vm_flags : 0)) {
2032                 gfp_t huge_gfp;
2033
2034                 huge_gfp = vma_thp_gfp_mask(vma);
2035                 huge_gfp = limit_gfp_mask(huge_gfp, gfp);
2036                 folio = shmem_alloc_and_add_folio(huge_gfp,
2037                                 inode, index, fault_mm, true);
2038                 if (!IS_ERR(folio)) {
2039                         count_vm_event(THP_FILE_ALLOC);
2040                         goto alloced;
2041                 }
2042                 if (PTR_ERR(folio) == -EEXIST)
2043                         goto repeat;
2044         }
2045
2046         folio = shmem_alloc_and_add_folio(gfp, inode, index, fault_mm, false);
2047         if (IS_ERR(folio)) {
2048                 error = PTR_ERR(folio);
2049                 if (error == -EEXIST)
2050                         goto repeat;
2051                 folio = NULL;
2052                 goto unlock;
2053         }
2054
2055 alloced:
2056         alloced = true;
2057         if (folio_test_pmd_mappable(folio) &&
2058             DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
2059                                         folio_next_index(folio) - 1) {
2060                 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2061                 struct shmem_inode_info *info = SHMEM_I(inode);
2062                 /*
2063                  * Part of the large folio is beyond i_size: subject
2064                  * to shrink under memory pressure.
2065                  */
2066                 spin_lock(&sbinfo->shrinklist_lock);
2067                 /*
2068                  * _careful to defend against unlocked access to
2069                  * ->shrink_list in shmem_unused_huge_shrink()
2070                  */
2071                 if (list_empty_careful(&info->shrinklist)) {
2072                         list_add_tail(&info->shrinklist,
2073                                       &sbinfo->shrinklist);
2074                         sbinfo->shrinklist_len++;
2075                 }
2076                 spin_unlock(&sbinfo->shrinklist_lock);
2077         }
2078
2079         if (sgp == SGP_WRITE)
2080                 folio_set_referenced(folio);
2081         /*
2082          * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio.
2083          */
2084         if (sgp == SGP_FALLOC)
2085                 sgp = SGP_WRITE;
2086 clear:
2087         /*
2088          * Let SGP_WRITE caller clear ends if write does not fill folio;
2089          * but SGP_FALLOC on a folio fallocated earlier must initialize
2090          * it now, lest undo on failure cancel our earlier guarantee.
2091          */
2092         if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) {
2093                 long i, n = folio_nr_pages(folio);
2094
2095                 for (i = 0; i < n; i++)
2096                         clear_highpage(folio_page(folio, i));
2097                 flush_dcache_folio(folio);
2098                 folio_mark_uptodate(folio);
2099         }
2100
2101         /* Perhaps the file has been truncated since we checked */
2102         if (sgp <= SGP_CACHE &&
2103             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
2104                 error = -EINVAL;
2105                 goto unlock;
2106         }
2107 out:
2108         *foliop = folio;
2109         return 0;
2110
2111         /*
2112          * Error recovery.
2113          */
2114 unlock:
2115         if (alloced)
2116                 filemap_remove_folio(folio);
2117         shmem_recalc_inode(inode, 0, 0);
2118         if (folio) {
2119                 folio_unlock(folio);
2120                 folio_put(folio);
2121         }
2122         return error;
2123 }
2124
2125 int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop,
2126                 enum sgp_type sgp)
2127 {
2128         return shmem_get_folio_gfp(inode, index, foliop, sgp,
2129                         mapping_gfp_mask(inode->i_mapping), NULL, NULL);
2130 }
2131
2132 /*
2133  * This is like autoremove_wake_function, but it removes the wait queue
2134  * entry unconditionally - even if something else had already woken the
2135  * target.
2136  */
2137 static int synchronous_wake_function(wait_queue_entry_t *wait,
2138                         unsigned int mode, int sync, void *key)
2139 {
2140         int ret = default_wake_function(wait, mode, sync, key);
2141         list_del_init(&wait->entry);
2142         return ret;
2143 }
2144
2145 /*
2146  * Trinity finds that probing a hole which tmpfs is punching can
2147  * prevent the hole-punch from ever completing: which in turn
2148  * locks writers out with its hold on i_rwsem.  So refrain from
2149  * faulting pages into the hole while it's being punched.  Although
2150  * shmem_undo_range() does remove the additions, it may be unable to
2151  * keep up, as each new page needs its own unmap_mapping_range() call,
2152  * and the i_mmap tree grows ever slower to scan if new vmas are added.
2153  *
2154  * It does not matter if we sometimes reach this check just before the
2155  * hole-punch begins, so that one fault then races with the punch:
2156  * we just need to make racing faults a rare case.
2157  *
2158  * The implementation below would be much simpler if we just used a
2159  * standard mutex or completion: but we cannot take i_rwsem in fault,
2160  * and bloating every shmem inode for this unlikely case would be sad.
2161  */
2162 static vm_fault_t shmem_falloc_wait(struct vm_fault *vmf, struct inode *inode)
2163 {
2164         struct shmem_falloc *shmem_falloc;
2165         struct file *fpin = NULL;
2166         vm_fault_t ret = 0;
2167
2168         spin_lock(&inode->i_lock);
2169         shmem_falloc = inode->i_private;
2170         if (shmem_falloc &&
2171             shmem_falloc->waitq &&
2172             vmf->pgoff >= shmem_falloc->start &&
2173             vmf->pgoff < shmem_falloc->next) {
2174                 wait_queue_head_t *shmem_falloc_waitq;
2175                 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2176
2177                 ret = VM_FAULT_NOPAGE;
2178                 fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2179                 shmem_falloc_waitq = shmem_falloc->waitq;
2180                 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2181                                 TASK_UNINTERRUPTIBLE);
2182                 spin_unlock(&inode->i_lock);
2183                 schedule();
2184
2185                 /*
2186                  * shmem_falloc_waitq points into the shmem_fallocate()
2187                  * stack of the hole-punching task: shmem_falloc_waitq
2188                  * is usually invalid by the time we reach here, but
2189                  * finish_wait() does not dereference it in that case;
2190                  * though i_lock needed lest racing with wake_up_all().
2191                  */
2192                 spin_lock(&inode->i_lock);
2193                 finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2194         }
2195         spin_unlock(&inode->i_lock);
2196         if (fpin) {
2197                 fput(fpin);
2198                 ret = VM_FAULT_RETRY;
2199         }
2200         return ret;
2201 }
2202
2203 static vm_fault_t shmem_fault(struct vm_fault *vmf)
2204 {
2205         struct inode *inode = file_inode(vmf->vma->vm_file);
2206         gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
2207         struct folio *folio = NULL;
2208         vm_fault_t ret = 0;
2209         int err;
2210
2211         /*
2212          * Trinity finds that probing a hole which tmpfs is punching can
2213          * prevent the hole-punch from ever completing: noted in i_private.
2214          */
2215         if (unlikely(inode->i_private)) {
2216                 ret = shmem_falloc_wait(vmf, inode);
2217                 if (ret)
2218                         return ret;
2219         }
2220
2221         WARN_ON_ONCE(vmf->page != NULL);
2222         err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE,
2223                                   gfp, vmf, &ret);
2224         if (err)
2225                 return vmf_error(err);
2226         if (folio) {
2227                 vmf->page = folio_file_page(folio, vmf->pgoff);
2228                 ret |= VM_FAULT_LOCKED;
2229         }
2230         return ret;
2231 }
2232
2233 unsigned long shmem_get_unmapped_area(struct file *file,
2234                                       unsigned long uaddr, unsigned long len,
2235                                       unsigned long pgoff, unsigned long flags)
2236 {
2237         unsigned long (*get_area)(struct file *,
2238                 unsigned long, unsigned long, unsigned long, unsigned long);
2239         unsigned long addr;
2240         unsigned long offset;
2241         unsigned long inflated_len;
2242         unsigned long inflated_addr;
2243         unsigned long inflated_offset;
2244
2245         if (len > TASK_SIZE)
2246                 return -ENOMEM;
2247
2248         get_area = current->mm->get_unmapped_area;
2249         addr = get_area(file, uaddr, len, pgoff, flags);
2250
2251         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2252                 return addr;
2253         if (IS_ERR_VALUE(addr))
2254                 return addr;
2255         if (addr & ~PAGE_MASK)
2256                 return addr;
2257         if (addr > TASK_SIZE - len)
2258                 return addr;
2259
2260         if (shmem_huge == SHMEM_HUGE_DENY)
2261                 return addr;
2262         if (len < HPAGE_PMD_SIZE)
2263                 return addr;
2264         if (flags & MAP_FIXED)
2265                 return addr;
2266         /*
2267          * Our priority is to support MAP_SHARED mapped hugely;
2268          * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2269          * But if caller specified an address hint and we allocated area there
2270          * successfully, respect that as before.
2271          */
2272         if (uaddr == addr)
2273                 return addr;
2274
2275         if (shmem_huge != SHMEM_HUGE_FORCE) {
2276                 struct super_block *sb;
2277
2278                 if (file) {
2279                         VM_BUG_ON(file->f_op != &shmem_file_operations);
2280                         sb = file_inode(file)->i_sb;
2281                 } else {
2282                         /*
2283                          * Called directly from mm/mmap.c, or drivers/char/mem.c
2284                          * for "/dev/zero", to create a shared anonymous object.
2285                          */
2286                         if (IS_ERR(shm_mnt))
2287                                 return addr;
2288                         sb = shm_mnt->mnt_sb;
2289                 }
2290                 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2291                         return addr;
2292         }
2293
2294         offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2295         if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2296                 return addr;
2297         if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2298                 return addr;
2299
2300         inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2301         if (inflated_len > TASK_SIZE)
2302                 return addr;
2303         if (inflated_len < len)
2304                 return addr;
2305
2306         inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags);
2307         if (IS_ERR_VALUE(inflated_addr))
2308                 return addr;
2309         if (inflated_addr & ~PAGE_MASK)
2310                 return addr;
2311
2312         inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2313         inflated_addr += offset - inflated_offset;
2314         if (inflated_offset > offset)
2315                 inflated_addr += HPAGE_PMD_SIZE;
2316
2317         if (inflated_addr > TASK_SIZE - len)
2318                 return addr;
2319         return inflated_addr;
2320 }
2321
2322 #ifdef CONFIG_NUMA
2323 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2324 {
2325         struct inode *inode = file_inode(vma->vm_file);
2326         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2327 }
2328
2329 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2330                                           unsigned long addr, pgoff_t *ilx)
2331 {
2332         struct inode *inode = file_inode(vma->vm_file);
2333         pgoff_t index;
2334
2335         /*
2336          * Bias interleave by inode number to distribute better across nodes;
2337          * but this interface is independent of which page order is used, so
2338          * supplies only that bias, letting caller apply the offset (adjusted
2339          * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()).
2340          */
2341         *ilx = inode->i_ino;
2342         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2343         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2344 }
2345
2346 static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info,
2347                         pgoff_t index, unsigned int order, pgoff_t *ilx)
2348 {
2349         struct mempolicy *mpol;
2350
2351         /* Bias interleave by inode number to distribute better across nodes */
2352         *ilx = info->vfs_inode.i_ino + (index >> order);
2353
2354         mpol = mpol_shared_policy_lookup(&info->policy, index);
2355         return mpol ? mpol : get_task_policy(current);
2356 }
2357 #else
2358 static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info,
2359                         pgoff_t index, unsigned int order, pgoff_t *ilx)
2360 {
2361         *ilx = 0;
2362         return NULL;
2363 }
2364 #endif /* CONFIG_NUMA */
2365
2366 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
2367 {
2368         struct inode *inode = file_inode(file);
2369         struct shmem_inode_info *info = SHMEM_I(inode);
2370         int retval = -ENOMEM;
2371
2372         /*
2373          * What serializes the accesses to info->flags?
2374          * ipc_lock_object() when called from shmctl_do_lock(),
2375          * no serialization needed when called from shm_destroy().
2376          */
2377         if (lock && !(info->flags & VM_LOCKED)) {
2378                 if (!user_shm_lock(inode->i_size, ucounts))
2379                         goto out_nomem;
2380                 info->flags |= VM_LOCKED;
2381                 mapping_set_unevictable(file->f_mapping);
2382         }
2383         if (!lock && (info->flags & VM_LOCKED) && ucounts) {
2384                 user_shm_unlock(inode->i_size, ucounts);
2385                 info->flags &= ~VM_LOCKED;
2386                 mapping_clear_unevictable(file->f_mapping);
2387         }
2388         retval = 0;
2389
2390 out_nomem:
2391         return retval;
2392 }
2393
2394 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2395 {
2396         struct inode *inode = file_inode(file);
2397         struct shmem_inode_info *info = SHMEM_I(inode);
2398         int ret;
2399
2400         ret = seal_check_write(info->seals, vma);
2401         if (ret)
2402                 return ret;
2403
2404         /* arm64 - allow memory tagging on RAM-based files */
2405         vm_flags_set(vma, VM_MTE_ALLOWED);
2406
2407         file_accessed(file);
2408         /* This is anonymous shared memory if it is unlinked at the time of mmap */
2409         if (inode->i_nlink)
2410                 vma->vm_ops = &shmem_vm_ops;
2411         else
2412                 vma->vm_ops = &shmem_anon_vm_ops;
2413         return 0;
2414 }
2415
2416 static int shmem_file_open(struct inode *inode, struct file *file)
2417 {
2418         file->f_mode |= FMODE_CAN_ODIRECT;
2419         return generic_file_open(inode, file);
2420 }
2421
2422 #ifdef CONFIG_TMPFS_XATTR
2423 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2424
2425 /*
2426  * chattr's fsflags are unrelated to extended attributes,
2427  * but tmpfs has chosen to enable them under the same config option.
2428  */
2429 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2430 {
2431         unsigned int i_flags = 0;
2432
2433         if (fsflags & FS_NOATIME_FL)
2434                 i_flags |= S_NOATIME;
2435         if (fsflags & FS_APPEND_FL)
2436                 i_flags |= S_APPEND;
2437         if (fsflags & FS_IMMUTABLE_FL)
2438                 i_flags |= S_IMMUTABLE;
2439         /*
2440          * But FS_NODUMP_FL does not require any action in i_flags.
2441          */
2442         inode_set_flags(inode, i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE);
2443 }
2444 #else
2445 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2446 {
2447 }
2448 #define shmem_initxattrs NULL
2449 #endif
2450
2451 static struct offset_ctx *shmem_get_offset_ctx(struct inode *inode)
2452 {
2453         return &SHMEM_I(inode)->dir_offsets;
2454 }
2455
2456 static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
2457                                              struct super_block *sb,
2458                                              struct inode *dir, umode_t mode,
2459                                              dev_t dev, unsigned long flags)
2460 {
2461         struct inode *inode;
2462         struct shmem_inode_info *info;
2463         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2464         ino_t ino;
2465         int err;
2466
2467         err = shmem_reserve_inode(sb, &ino);
2468         if (err)
2469                 return ERR_PTR(err);
2470
2471         inode = new_inode(sb);
2472         if (!inode) {
2473                 shmem_free_inode(sb, 0);
2474                 return ERR_PTR(-ENOSPC);
2475         }
2476
2477         inode->i_ino = ino;
2478         inode_init_owner(idmap, inode, dir, mode);
2479         inode->i_blocks = 0;
2480         simple_inode_init_ts(inode);
2481         inode->i_generation = get_random_u32();
2482         info = SHMEM_I(inode);
2483         memset(info, 0, (char *)inode - (char *)info);
2484         spin_lock_init(&info->lock);
2485         atomic_set(&info->stop_eviction, 0);
2486         info->seals = F_SEAL_SEAL;
2487         info->flags = flags & VM_NORESERVE;
2488         info->i_crtime = inode_get_mtime(inode);
2489         info->fsflags = (dir == NULL) ? 0 :
2490                 SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED;
2491         if (info->fsflags)
2492                 shmem_set_inode_flags(inode, info->fsflags);
2493         INIT_LIST_HEAD(&info->shrinklist);
2494         INIT_LIST_HEAD(&info->swaplist);
2495         simple_xattrs_init(&info->xattrs);
2496         cache_no_acl(inode);
2497         if (sbinfo->noswap)
2498                 mapping_set_unevictable(inode->i_mapping);
2499         mapping_set_large_folios(inode->i_mapping);
2500
2501         switch (mode & S_IFMT) {
2502         default:
2503                 inode->i_op = &shmem_special_inode_operations;
2504                 init_special_inode(inode, mode, dev);
2505                 break;
2506         case S_IFREG:
2507                 inode->i_mapping->a_ops = &shmem_aops;
2508                 inode->i_op = &shmem_inode_operations;
2509                 inode->i_fop = &shmem_file_operations;
2510                 mpol_shared_policy_init(&info->policy,
2511                                          shmem_get_sbmpol(sbinfo));
2512                 break;
2513         case S_IFDIR:
2514                 inc_nlink(inode);
2515                 /* Some things misbehave if size == 0 on a directory */
2516                 inode->i_size = 2 * BOGO_DIRENT_SIZE;
2517                 inode->i_op = &shmem_dir_inode_operations;
2518                 inode->i_fop = &simple_offset_dir_operations;
2519                 simple_offset_init(shmem_get_offset_ctx(inode));
2520                 break;
2521         case S_IFLNK:
2522                 /*
2523                  * Must not load anything in the rbtree,
2524                  * mpol_free_shared_policy will not be called.
2525                  */
2526                 mpol_shared_policy_init(&info->policy, NULL);
2527                 break;
2528         }
2529
2530         lockdep_annotate_inode_mutex_key(inode);
2531         return inode;
2532 }
2533
2534 #ifdef CONFIG_TMPFS_QUOTA
2535 static struct inode *shmem_get_inode(struct mnt_idmap *idmap,
2536                                      struct super_block *sb, struct inode *dir,
2537                                      umode_t mode, dev_t dev, unsigned long flags)
2538 {
2539         int err;
2540         struct inode *inode;
2541
2542         inode = __shmem_get_inode(idmap, sb, dir, mode, dev, flags);
2543         if (IS_ERR(inode))
2544                 return inode;
2545
2546         err = dquot_initialize(inode);
2547         if (err)
2548                 goto errout;
2549
2550         err = dquot_alloc_inode(inode);
2551         if (err) {
2552                 dquot_drop(inode);
2553                 goto errout;
2554         }
2555         return inode;
2556
2557 errout:
2558         inode->i_flags |= S_NOQUOTA;
2559         iput(inode);
2560         return ERR_PTR(err);
2561 }
2562 #else
2563 static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap,
2564                                      struct super_block *sb, struct inode *dir,
2565                                      umode_t mode, dev_t dev, unsigned long flags)
2566 {
2567         return __shmem_get_inode(idmap, sb, dir, mode, dev, flags);
2568 }
2569 #endif /* CONFIG_TMPFS_QUOTA */
2570
2571 #ifdef CONFIG_USERFAULTFD
2572 int shmem_mfill_atomic_pte(pmd_t *dst_pmd,
2573                            struct vm_area_struct *dst_vma,
2574                            unsigned long dst_addr,
2575                            unsigned long src_addr,
2576                            uffd_flags_t flags,
2577                            struct folio **foliop)
2578 {
2579         struct inode *inode = file_inode(dst_vma->vm_file);
2580         struct shmem_inode_info *info = SHMEM_I(inode);
2581         struct address_space *mapping = inode->i_mapping;
2582         gfp_t gfp = mapping_gfp_mask(mapping);
2583         pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2584         void *page_kaddr;
2585         struct folio *folio;
2586         int ret;
2587         pgoff_t max_off;
2588
2589         if (shmem_inode_acct_blocks(inode, 1)) {
2590                 /*
2591                  * We may have got a page, returned -ENOENT triggering a retry,
2592                  * and now we find ourselves with -ENOMEM. Release the page, to
2593                  * avoid a BUG_ON in our caller.
2594                  */
2595                 if (unlikely(*foliop)) {
2596                         folio_put(*foliop);
2597                         *foliop = NULL;
2598                 }
2599                 return -ENOMEM;
2600         }
2601
2602         if (!*foliop) {
2603                 ret = -ENOMEM;
2604                 folio = shmem_alloc_folio(gfp, info, pgoff);
2605                 if (!folio)
2606                         goto out_unacct_blocks;
2607
2608                 if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) {
2609                         page_kaddr = kmap_local_folio(folio, 0);
2610                         /*
2611                          * The read mmap_lock is held here.  Despite the
2612                          * mmap_lock being read recursive a deadlock is still
2613                          * possible if a writer has taken a lock.  For example:
2614                          *
2615                          * process A thread 1 takes read lock on own mmap_lock
2616                          * process A thread 2 calls mmap, blocks taking write lock
2617                          * process B thread 1 takes page fault, read lock on own mmap lock
2618                          * process B thread 2 calls mmap, blocks taking write lock
2619                          * process A thread 1 blocks taking read lock on process B
2620                          * process B thread 1 blocks taking read lock on process A
2621                          *
2622                          * Disable page faults to prevent potential deadlock
2623                          * and retry the copy outside the mmap_lock.
2624                          */
2625                         pagefault_disable();
2626                         ret = copy_from_user(page_kaddr,
2627                                              (const void __user *)src_addr,
2628                                              PAGE_SIZE);
2629                         pagefault_enable();
2630                         kunmap_local(page_kaddr);
2631
2632                         /* fallback to copy_from_user outside mmap_lock */
2633                         if (unlikely(ret)) {
2634                                 *foliop = folio;
2635                                 ret = -ENOENT;
2636                                 /* don't free the page */
2637                                 goto out_unacct_blocks;
2638                         }
2639
2640                         flush_dcache_folio(folio);
2641                 } else {                /* ZEROPAGE */
2642                         clear_user_highpage(&folio->page, dst_addr);
2643                 }
2644         } else {
2645                 folio = *foliop;
2646                 VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
2647                 *foliop = NULL;
2648         }
2649
2650         VM_BUG_ON(folio_test_locked(folio));
2651         VM_BUG_ON(folio_test_swapbacked(folio));
2652         __folio_set_locked(folio);
2653         __folio_set_swapbacked(folio);
2654         __folio_mark_uptodate(folio);
2655
2656         ret = -EFAULT;
2657         max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2658         if (unlikely(pgoff >= max_off))
2659                 goto out_release;
2660
2661         ret = mem_cgroup_charge(folio, dst_vma->vm_mm, gfp);
2662         if (ret)
2663                 goto out_release;
2664         ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL, gfp);
2665         if (ret)
2666                 goto out_release;
2667
2668         ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,
2669                                        &folio->page, true, flags);
2670         if (ret)
2671                 goto out_delete_from_cache;
2672
2673         shmem_recalc_inode(inode, 1, 0);
2674         folio_unlock(folio);
2675         return 0;
2676 out_delete_from_cache:
2677         filemap_remove_folio(folio);
2678 out_release:
2679         folio_unlock(folio);
2680         folio_put(folio);
2681 out_unacct_blocks:
2682         shmem_inode_unacct_blocks(inode, 1);
2683         return ret;
2684 }
2685 #endif /* CONFIG_USERFAULTFD */
2686
2687 #ifdef CONFIG_TMPFS
2688 static const struct inode_operations shmem_symlink_inode_operations;
2689 static const struct inode_operations shmem_short_symlink_operations;
2690
2691 static int
2692 shmem_write_begin(struct file *file, struct address_space *mapping,
2693                         loff_t pos, unsigned len,
2694                         struct page **pagep, void **fsdata)
2695 {
2696         struct inode *inode = mapping->host;
2697         struct shmem_inode_info *info = SHMEM_I(inode);
2698         pgoff_t index = pos >> PAGE_SHIFT;
2699         struct folio *folio;
2700         int ret = 0;
2701
2702         /* i_rwsem is held by caller */
2703         if (unlikely(info->seals & (F_SEAL_GROW |
2704                                    F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
2705                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
2706                         return -EPERM;
2707                 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2708                         return -EPERM;
2709         }
2710
2711         ret = shmem_get_folio(inode, index, &folio, SGP_WRITE);
2712         if (ret)
2713                 return ret;
2714
2715         *pagep = folio_file_page(folio, index);
2716         if (PageHWPoison(*pagep)) {
2717                 folio_unlock(folio);
2718                 folio_put(folio);
2719                 *pagep = NULL;
2720                 return -EIO;
2721         }
2722
2723         return 0;
2724 }
2725
2726 static int
2727 shmem_write_end(struct file *file, struct address_space *mapping,
2728                         loff_t pos, unsigned len, unsigned copied,
2729                         struct page *page, void *fsdata)
2730 {
2731         struct folio *folio = page_folio(page);
2732         struct inode *inode = mapping->host;
2733
2734         if (pos + copied > inode->i_size)
2735                 i_size_write(inode, pos + copied);
2736
2737         if (!folio_test_uptodate(folio)) {
2738                 if (copied < folio_size(folio)) {
2739                         size_t from = offset_in_folio(folio, pos);
2740                         folio_zero_segments(folio, 0, from,
2741                                         from + copied, folio_size(folio));
2742                 }
2743                 folio_mark_uptodate(folio);
2744         }
2745         folio_mark_dirty(folio);
2746         folio_unlock(folio);
2747         folio_put(folio);
2748
2749         return copied;
2750 }
2751
2752 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2753 {
2754         struct file *file = iocb->ki_filp;
2755         struct inode *inode = file_inode(file);
2756         struct address_space *mapping = inode->i_mapping;
2757         pgoff_t index;
2758         unsigned long offset;
2759         int error = 0;
2760         ssize_t retval = 0;
2761         loff_t *ppos = &iocb->ki_pos;
2762
2763         index = *ppos >> PAGE_SHIFT;
2764         offset = *ppos & ~PAGE_MASK;
2765
2766         for (;;) {
2767                 struct folio *folio = NULL;
2768                 struct page *page = NULL;
2769                 pgoff_t end_index;
2770                 unsigned long nr, ret;
2771                 loff_t i_size = i_size_read(inode);
2772
2773                 end_index = i_size >> PAGE_SHIFT;
2774                 if (index > end_index)
2775                         break;
2776                 if (index == end_index) {
2777                         nr = i_size & ~PAGE_MASK;
2778                         if (nr <= offset)
2779                                 break;
2780                 }
2781
2782                 error = shmem_get_folio(inode, index, &folio, SGP_READ);
2783                 if (error) {
2784                         if (error == -EINVAL)
2785                                 error = 0;
2786                         break;
2787                 }
2788                 if (folio) {
2789                         folio_unlock(folio);
2790
2791                         page = folio_file_page(folio, index);
2792                         if (PageHWPoison(page)) {
2793                                 folio_put(folio);
2794                                 error = -EIO;
2795                                 break;
2796                         }
2797                 }
2798
2799                 /*
2800                  * We must evaluate after, since reads (unlike writes)
2801                  * are called without i_rwsem protection against truncate
2802                  */
2803                 nr = PAGE_SIZE;
2804                 i_size = i_size_read(inode);
2805                 end_index = i_size >> PAGE_SHIFT;
2806                 if (index == end_index) {
2807                         nr = i_size & ~PAGE_MASK;
2808                         if (nr <= offset) {
2809                                 if (folio)
2810                                         folio_put(folio);
2811                                 break;
2812                         }
2813                 }
2814                 nr -= offset;
2815
2816                 if (folio) {
2817                         /*
2818                          * If users can be writing to this page using arbitrary
2819                          * virtual addresses, take care about potential aliasing
2820                          * before reading the page on the kernel side.
2821                          */
2822                         if (mapping_writably_mapped(mapping))
2823                                 flush_dcache_page(page);
2824                         /*
2825                          * Mark the page accessed if we read the beginning.
2826                          */
2827                         if (!offset)
2828                                 folio_mark_accessed(folio);
2829                         /*
2830                          * Ok, we have the page, and it's up-to-date, so
2831                          * now we can copy it to user space...
2832                          */
2833                         ret = copy_page_to_iter(page, offset, nr, to);
2834                         folio_put(folio);
2835
2836                 } else if (user_backed_iter(to)) {
2837                         /*
2838                          * Copy to user tends to be so well optimized, but
2839                          * clear_user() not so much, that it is noticeably
2840                          * faster to copy the zero page instead of clearing.
2841                          */
2842                         ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
2843                 } else {
2844                         /*
2845                          * But submitting the same page twice in a row to
2846                          * splice() - or others? - can result in confusion:
2847                          * so don't attempt that optimization on pipes etc.
2848                          */
2849                         ret = iov_iter_zero(nr, to);
2850                 }
2851
2852                 retval += ret;
2853                 offset += ret;
2854                 index += offset >> PAGE_SHIFT;
2855                 offset &= ~PAGE_MASK;
2856
2857                 if (!iov_iter_count(to))
2858                         break;
2859                 if (ret < nr) {
2860                         error = -EFAULT;
2861                         break;
2862                 }
2863                 cond_resched();
2864         }
2865
2866         *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2867         file_accessed(file);
2868         return retval ? retval : error;
2869 }
2870
2871 static ssize_t shmem_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
2872 {
2873         struct file *file = iocb->ki_filp;
2874         struct inode *inode = file->f_mapping->host;
2875         ssize_t ret;
2876
2877         inode_lock(inode);
2878         ret = generic_write_checks(iocb, from);
2879         if (ret <= 0)
2880                 goto unlock;
2881         ret = file_remove_privs(file);
2882         if (ret)
2883                 goto unlock;
2884         ret = file_update_time(file);
2885         if (ret)
2886                 goto unlock;
2887         ret = generic_perform_write(iocb, from);
2888 unlock:
2889         inode_unlock(inode);
2890         return ret;
2891 }
2892
2893 static bool zero_pipe_buf_get(struct pipe_inode_info *pipe,
2894                               struct pipe_buffer *buf)
2895 {
2896         return true;
2897 }
2898
2899 static void zero_pipe_buf_release(struct pipe_inode_info *pipe,
2900                                   struct pipe_buffer *buf)
2901 {
2902 }
2903
2904 static bool zero_pipe_buf_try_steal(struct pipe_inode_info *pipe,
2905                                     struct pipe_buffer *buf)
2906 {
2907         return false;
2908 }
2909
2910 static const struct pipe_buf_operations zero_pipe_buf_ops = {
2911         .release        = zero_pipe_buf_release,
2912         .try_steal      = zero_pipe_buf_try_steal,
2913         .get            = zero_pipe_buf_get,
2914 };
2915
2916 static size_t splice_zeropage_into_pipe(struct pipe_inode_info *pipe,
2917                                         loff_t fpos, size_t size)
2918 {
2919         size_t offset = fpos & ~PAGE_MASK;
2920
2921         size = min_t(size_t, size, PAGE_SIZE - offset);
2922
2923         if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
2924                 struct pipe_buffer *buf = pipe_head_buf(pipe);
2925
2926                 *buf = (struct pipe_buffer) {
2927                         .ops    = &zero_pipe_buf_ops,
2928                         .page   = ZERO_PAGE(0),
2929                         .offset = offset,
2930                         .len    = size,
2931                 };
2932                 pipe->head++;
2933         }
2934
2935         return size;
2936 }
2937
2938 static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
2939                                       struct pipe_inode_info *pipe,
2940                                       size_t len, unsigned int flags)
2941 {
2942         struct inode *inode = file_inode(in);
2943         struct address_space *mapping = inode->i_mapping;
2944         struct folio *folio = NULL;
2945         size_t total_spliced = 0, used, npages, n, part;
2946         loff_t isize;
2947         int error = 0;
2948
2949         /* Work out how much data we can actually add into the pipe */
2950         used = pipe_occupancy(pipe->head, pipe->tail);
2951         npages = max_t(ssize_t, pipe->max_usage - used, 0);
2952         len = min_t(size_t, len, npages * PAGE_SIZE);
2953
2954         do {
2955                 if (*ppos >= i_size_read(inode))
2956                         break;
2957
2958                 error = shmem_get_folio(inode, *ppos / PAGE_SIZE, &folio,
2959                                         SGP_READ);
2960                 if (error) {
2961                         if (error == -EINVAL)
2962                                 error = 0;
2963                         break;
2964                 }
2965                 if (folio) {
2966                         folio_unlock(folio);
2967
2968                         if (folio_test_hwpoison(folio) ||
2969                             (folio_test_large(folio) &&
2970                              folio_test_has_hwpoisoned(folio))) {
2971                                 error = -EIO;
2972                                 break;
2973                         }
2974                 }
2975
2976                 /*
2977                  * i_size must be checked after we know the pages are Uptodate.
2978                  *
2979                  * Checking i_size after the check allows us to calculate
2980                  * the correct value for "nr", which means the zero-filled
2981                  * part of the page is not copied back to userspace (unless
2982                  * another truncate extends the file - this is desired though).
2983                  */
2984                 isize = i_size_read(inode);
2985                 if (unlikely(*ppos >= isize))
2986                         break;
2987                 part = min_t(loff_t, isize - *ppos, len);
2988
2989                 if (folio) {
2990                         /*
2991                          * If users can be writing to this page using arbitrary
2992                          * virtual addresses, take care about potential aliasing
2993                          * before reading the page on the kernel side.
2994                          */
2995                         if (mapping_writably_mapped(mapping))
2996                                 flush_dcache_folio(folio);
2997                         folio_mark_accessed(folio);
2998                         /*
2999                          * Ok, we have the page, and it's up-to-date, so we can
3000                          * now splice it into the pipe.
3001                          */
3002                         n = splice_folio_into_pipe(pipe, folio, *ppos, part);
3003                         folio_put(folio);
3004                         folio = NULL;
3005                 } else {
3006                         n = splice_zeropage_into_pipe(pipe, *ppos, part);
3007                 }
3008
3009                 if (!n)
3010                         break;
3011                 len -= n;
3012                 total_spliced += n;
3013                 *ppos += n;
3014                 in->f_ra.prev_pos = *ppos;
3015                 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
3016                         break;
3017
3018                 cond_resched();
3019         } while (len);
3020
3021         if (folio)
3022                 folio_put(folio);
3023
3024         file_accessed(in);
3025         return total_spliced ? total_spliced : error;
3026 }
3027
3028 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
3029 {
3030         struct address_space *mapping = file->f_mapping;
3031         struct inode *inode = mapping->host;
3032
3033         if (whence != SEEK_DATA && whence != SEEK_HOLE)
3034                 return generic_file_llseek_size(file, offset, whence,
3035                                         MAX_LFS_FILESIZE, i_size_read(inode));
3036         if (offset < 0)
3037                 return -ENXIO;
3038
3039         inode_lock(inode);
3040         /* We're holding i_rwsem so we can access i_size directly */
3041         offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence);
3042         if (offset >= 0)
3043                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
3044         inode_unlock(inode);
3045         return offset;
3046 }
3047
3048 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
3049                                                          loff_t len)
3050 {
3051         struct inode *inode = file_inode(file);
3052         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3053         struct shmem_inode_info *info = SHMEM_I(inode);
3054         struct shmem_falloc shmem_falloc;
3055         pgoff_t start, index, end, undo_fallocend;
3056         int error;
3057
3058         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
3059                 return -EOPNOTSUPP;
3060
3061         inode_lock(inode);
3062
3063         if (mode & FALLOC_FL_PUNCH_HOLE) {
3064                 struct address_space *mapping = file->f_mapping;
3065                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
3066                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
3067                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
3068
3069                 /* protected by i_rwsem */
3070                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
3071                         error = -EPERM;
3072                         goto out;
3073                 }
3074
3075                 shmem_falloc.waitq = &shmem_falloc_waitq;
3076                 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
3077                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
3078                 spin_lock(&inode->i_lock);
3079                 inode->i_private = &shmem_falloc;
3080                 spin_unlock(&inode->i_lock);
3081
3082                 if ((u64)unmap_end > (u64)unmap_start)
3083                         unmap_mapping_range(mapping, unmap_start,
3084                                             1 + unmap_end - unmap_start, 0);
3085                 shmem_truncate_range(inode, offset, offset + len - 1);
3086                 /* No need to unmap again: hole-punching leaves COWed pages */
3087
3088                 spin_lock(&inode->i_lock);
3089                 inode->i_private = NULL;
3090                 wake_up_all(&shmem_falloc_waitq);
3091                 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
3092                 spin_unlock(&inode->i_lock);
3093                 error = 0;
3094                 goto out;
3095         }
3096
3097         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
3098         error = inode_newsize_ok(inode, offset + len);
3099         if (error)
3100                 goto out;
3101
3102         if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
3103                 error = -EPERM;
3104                 goto out;
3105         }
3106
3107         start = offset >> PAGE_SHIFT;
3108         end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
3109         /* Try to avoid a swapstorm if len is impossible to satisfy */
3110         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
3111                 error = -ENOSPC;
3112                 goto out;
3113         }
3114
3115         shmem_falloc.waitq = NULL;
3116         shmem_falloc.start = start;
3117         shmem_falloc.next  = start;
3118         shmem_falloc.nr_falloced = 0;
3119         shmem_falloc.nr_unswapped = 0;
3120         spin_lock(&inode->i_lock);
3121         inode->i_private = &shmem_falloc;
3122         spin_unlock(&inode->i_lock);
3123
3124         /*
3125          * info->fallocend is only relevant when huge pages might be
3126          * involved: to prevent split_huge_page() freeing fallocated
3127          * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size.
3128          */
3129         undo_fallocend = info->fallocend;
3130         if (info->fallocend < end)
3131                 info->fallocend = end;
3132
3133         for (index = start; index < end; ) {
3134                 struct folio *folio;
3135
3136                 /*
3137                  * Good, the fallocate(2) manpage permits EINTR: we may have
3138                  * been interrupted because we are using up too much memory.
3139                  */
3140                 if (signal_pending(current))
3141                         error = -EINTR;
3142                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
3143                         error = -ENOMEM;
3144                 else
3145                         error = shmem_get_folio(inode, index, &folio,
3146                                                 SGP_FALLOC);
3147                 if (error) {
3148                         info->fallocend = undo_fallocend;
3149                         /* Remove the !uptodate folios we added */
3150                         if (index > start) {
3151                                 shmem_undo_range(inode,
3152                                     (loff_t)start << PAGE_SHIFT,
3153                                     ((loff_t)index << PAGE_SHIFT) - 1, true);
3154                         }
3155                         goto undone;
3156                 }
3157
3158                 /*
3159                  * Here is a more important optimization than it appears:
3160                  * a second SGP_FALLOC on the same large folio will clear it,
3161                  * making it uptodate and un-undoable if we fail later.
3162                  */
3163                 index = folio_next_index(folio);
3164                 /* Beware 32-bit wraparound */
3165                 if (!index)
3166                         index--;
3167
3168                 /*
3169                  * Inform shmem_writepage() how far we have reached.
3170                  * No need for lock or barrier: we have the page lock.
3171                  */
3172                 if (!folio_test_uptodate(folio))
3173                         shmem_falloc.nr_falloced += index - shmem_falloc.next;
3174                 shmem_falloc.next = index;
3175
3176                 /*
3177                  * If !uptodate, leave it that way so that freeable folios
3178                  * can be recognized if we need to rollback on error later.
3179                  * But mark it dirty so that memory pressure will swap rather
3180                  * than free the folios we are allocating (and SGP_CACHE folios
3181                  * might still be clean: we now need to mark those dirty too).
3182                  */
3183                 folio_mark_dirty(folio);
3184                 folio_unlock(folio);
3185                 folio_put(folio);
3186                 cond_resched();
3187         }
3188
3189         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
3190                 i_size_write(inode, offset + len);
3191 undone:
3192         spin_lock(&inode->i_lock);
3193         inode->i_private = NULL;
3194         spin_unlock(&inode->i_lock);
3195 out:
3196         if (!error)
3197                 file_modified(file);
3198         inode_unlock(inode);
3199         return error;
3200 }
3201
3202 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
3203 {
3204         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
3205
3206         buf->f_type = TMPFS_MAGIC;
3207         buf->f_bsize = PAGE_SIZE;
3208         buf->f_namelen = NAME_MAX;
3209         if (sbinfo->max_blocks) {
3210                 buf->f_blocks = sbinfo->max_blocks;
3211                 buf->f_bavail =
3212                 buf->f_bfree  = sbinfo->max_blocks -
3213                                 percpu_counter_sum(&sbinfo->used_blocks);
3214         }
3215         if (sbinfo->max_inodes) {
3216                 buf->f_files = sbinfo->max_inodes;
3217                 buf->f_ffree = sbinfo->free_ispace / BOGO_INODE_SIZE;
3218         }
3219         /* else leave those fields 0 like simple_statfs */
3220
3221         buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b);
3222
3223         return 0;
3224 }
3225
3226 /*
3227  * File creation. Allocate an inode, and we're done..
3228  */
3229 static int
3230 shmem_mknod(struct mnt_idmap *idmap, struct inode *dir,
3231             struct dentry *dentry, umode_t mode, dev_t dev)
3232 {
3233         struct inode *inode;
3234         int error;
3235
3236         inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, dev, VM_NORESERVE);
3237         if (IS_ERR(inode))
3238                 return PTR_ERR(inode);
3239
3240         error = simple_acl_create(dir, inode);
3241         if (error)
3242                 goto out_iput;
3243         error = security_inode_init_security(inode, dir, &dentry->d_name,
3244                                              shmem_initxattrs, NULL);
3245         if (error && error != -EOPNOTSUPP)
3246                 goto out_iput;
3247
3248         error = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
3249         if (error)
3250                 goto out_iput;
3251
3252         dir->i_size += BOGO_DIRENT_SIZE;
3253         inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
3254         inode_inc_iversion(dir);
3255         d_instantiate(dentry, inode);
3256         dget(dentry); /* Extra count - pin the dentry in core */
3257         return error;
3258
3259 out_iput:
3260         iput(inode);
3261         return error;
3262 }
3263
3264 static int
3265 shmem_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
3266               struct file *file, umode_t mode)
3267 {
3268         struct inode *inode;
3269         int error;
3270
3271         inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, 0, VM_NORESERVE);
3272         if (IS_ERR(inode)) {
3273                 error = PTR_ERR(inode);
3274                 goto err_out;
3275         }
3276         error = security_inode_init_security(inode, dir, NULL,
3277                                              shmem_initxattrs, NULL);
3278         if (error && error != -EOPNOTSUPP)
3279                 goto out_iput;
3280         error = simple_acl_create(dir, inode);
3281         if (error)
3282                 goto out_iput;
3283         d_tmpfile(file, inode);
3284
3285 err_out:
3286         return finish_open_simple(file, error);
3287 out_iput:
3288         iput(inode);
3289         return error;
3290 }
3291
3292 static int shmem_mkdir(struct mnt_idmap *idmap, struct inode *dir,
3293                        struct dentry *dentry, umode_t mode)
3294 {
3295         int error;
3296
3297         error = shmem_mknod(idmap, dir, dentry, mode | S_IFDIR, 0);
3298         if (error)
3299                 return error;
3300         inc_nlink(dir);
3301         return 0;
3302 }
3303
3304 static int shmem_create(struct mnt_idmap *idmap, struct inode *dir,
3305                         struct dentry *dentry, umode_t mode, bool excl)
3306 {
3307         return shmem_mknod(idmap, dir, dentry, mode | S_IFREG, 0);
3308 }
3309
3310 /*
3311  * Link a file..
3312  */
3313 static int shmem_link(struct dentry *old_dentry, struct inode *dir,
3314                       struct dentry *dentry)
3315 {
3316         struct inode *inode = d_inode(old_dentry);
3317         int ret = 0;
3318
3319         /*
3320          * No ordinary (disk based) filesystem counts links as inodes;
3321          * but each new link needs a new dentry, pinning lowmem, and
3322          * tmpfs dentries cannot be pruned until they are unlinked.
3323          * But if an O_TMPFILE file is linked into the tmpfs, the
3324          * first link must skip that, to get the accounting right.
3325          */
3326         if (inode->i_nlink) {
3327                 ret = shmem_reserve_inode(inode->i_sb, NULL);
3328                 if (ret)
3329                         goto out;
3330         }
3331
3332         ret = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
3333         if (ret) {
3334                 if (inode->i_nlink)
3335                         shmem_free_inode(inode->i_sb, 0);
3336                 goto out;
3337         }
3338
3339         dir->i_size += BOGO_DIRENT_SIZE;
3340         inode_set_mtime_to_ts(dir,
3341                               inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode)));
3342         inode_inc_iversion(dir);
3343         inc_nlink(inode);
3344         ihold(inode);   /* New dentry reference */
3345         dget(dentry);   /* Extra pinning count for the created dentry */
3346         d_instantiate(dentry, inode);
3347 out:
3348         return ret;
3349 }
3350
3351 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
3352 {
3353         struct inode *inode = d_inode(dentry);
3354
3355         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
3356                 shmem_free_inode(inode->i_sb, 0);
3357
3358         simple_offset_remove(shmem_get_offset_ctx(dir), dentry);
3359
3360         dir->i_size -= BOGO_DIRENT_SIZE;
3361         inode_set_mtime_to_ts(dir,
3362                               inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode)));
3363         inode_inc_iversion(dir);
3364         drop_nlink(inode);
3365         dput(dentry);   /* Undo the count from "create" - does all the work */
3366         return 0;
3367 }
3368
3369 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
3370 {
3371         if (!simple_empty(dentry))
3372                 return -ENOTEMPTY;
3373
3374         drop_nlink(d_inode(dentry));
3375         drop_nlink(dir);
3376         return shmem_unlink(dir, dentry);
3377 }
3378
3379 static int shmem_whiteout(struct mnt_idmap *idmap,
3380                           struct inode *old_dir, struct dentry *old_dentry)
3381 {
3382         struct dentry *whiteout;
3383         int error;
3384
3385         whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3386         if (!whiteout)
3387                 return -ENOMEM;
3388
3389         error = shmem_mknod(idmap, old_dir, whiteout,
3390                             S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3391         dput(whiteout);
3392         if (error)
3393                 return error;
3394
3395         /*
3396          * Cheat and hash the whiteout while the old dentry is still in
3397          * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3398          *
3399          * d_lookup() will consistently find one of them at this point,
3400          * not sure which one, but that isn't even important.
3401          */
3402         d_rehash(whiteout);
3403         return 0;
3404 }
3405
3406 /*
3407  * The VFS layer already does all the dentry stuff for rename,
3408  * we just have to decrement the usage count for the target if
3409  * it exists so that the VFS layer correctly free's it when it
3410  * gets overwritten.
3411  */
3412 static int shmem_rename2(struct mnt_idmap *idmap,
3413                          struct inode *old_dir, struct dentry *old_dentry,
3414                          struct inode *new_dir, struct dentry *new_dentry,
3415                          unsigned int flags)
3416 {
3417         struct inode *inode = d_inode(old_dentry);
3418         int they_are_dirs = S_ISDIR(inode->i_mode);
3419         int error;
3420
3421         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3422                 return -EINVAL;
3423
3424         if (flags & RENAME_EXCHANGE)
3425                 return simple_offset_rename_exchange(old_dir, old_dentry,
3426                                                      new_dir, new_dentry);
3427
3428         if (!simple_empty(new_dentry))
3429                 return -ENOTEMPTY;
3430
3431         if (flags & RENAME_WHITEOUT) {
3432                 error = shmem_whiteout(idmap, old_dir, old_dentry);
3433                 if (error)
3434                         return error;
3435         }
3436
3437         simple_offset_remove(shmem_get_offset_ctx(old_dir), old_dentry);
3438         error = simple_offset_add(shmem_get_offset_ctx(new_dir), old_dentry);
3439         if (error)
3440                 return error;
3441
3442         if (d_really_is_positive(new_dentry)) {
3443                 (void) shmem_unlink(new_dir, new_dentry);
3444                 if (they_are_dirs) {
3445                         drop_nlink(d_inode(new_dentry));
3446                         drop_nlink(old_dir);
3447                 }
3448         } else if (they_are_dirs) {
3449                 drop_nlink(old_dir);
3450                 inc_nlink(new_dir);
3451         }
3452
3453         old_dir->i_size -= BOGO_DIRENT_SIZE;
3454         new_dir->i_size += BOGO_DIRENT_SIZE;
3455         simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
3456         inode_inc_iversion(old_dir);
3457         inode_inc_iversion(new_dir);
3458         return 0;
3459 }
3460
3461 static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir,
3462                          struct dentry *dentry, const char *symname)
3463 {
3464         int error;
3465         int len;
3466         struct inode *inode;
3467         struct folio *folio;
3468
3469         len = strlen(symname) + 1;
3470         if (len > PAGE_SIZE)
3471                 return -ENAMETOOLONG;
3472
3473         inode = shmem_get_inode(idmap, dir->i_sb, dir, S_IFLNK | 0777, 0,
3474                                 VM_NORESERVE);
3475         if (IS_ERR(inode))
3476                 return PTR_ERR(inode);
3477
3478         error = security_inode_init_security(inode, dir, &dentry->d_name,
3479                                              shmem_initxattrs, NULL);
3480         if (error && error != -EOPNOTSUPP)
3481                 goto out_iput;
3482
3483         error = simple_offset_add(shmem_get_offset_ctx(dir), dentry);
3484         if (error)
3485                 goto out_iput;
3486
3487         inode->i_size = len-1;
3488         if (len <= SHORT_SYMLINK_LEN) {
3489                 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3490                 if (!inode->i_link) {
3491                         error = -ENOMEM;
3492                         goto out_remove_offset;
3493                 }
3494                 inode->i_op = &shmem_short_symlink_operations;
3495         } else {
3496                 inode_nohighmem(inode);
3497                 error = shmem_get_folio(inode, 0, &folio, SGP_WRITE);
3498                 if (error)
3499                         goto out_remove_offset;
3500                 inode->i_mapping->a_ops = &shmem_aops;
3501                 inode->i_op = &shmem_symlink_inode_operations;
3502                 memcpy(folio_address(folio), symname, len);
3503                 folio_mark_uptodate(folio);
3504                 folio_mark_dirty(folio);
3505                 folio_unlock(folio);
3506                 folio_put(folio);
3507         }
3508         dir->i_size += BOGO_DIRENT_SIZE;
3509         inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
3510         inode_inc_iversion(dir);
3511         d_instantiate(dentry, inode);
3512         dget(dentry);
3513         return 0;
3514
3515 out_remove_offset:
3516         simple_offset_remove(shmem_get_offset_ctx(dir), dentry);
3517 out_iput:
3518         iput(inode);
3519         return error;
3520 }
3521
3522 static void shmem_put_link(void *arg)
3523 {
3524         folio_mark_accessed(arg);
3525         folio_put(arg);
3526 }
3527
3528 static const char *shmem_get_link(struct dentry *dentry, struct inode *inode,
3529                                   struct delayed_call *done)
3530 {
3531         struct folio *folio = NULL;
3532         int error;
3533
3534         if (!dentry) {
3535                 folio = filemap_get_folio(inode->i_mapping, 0);
3536                 if (IS_ERR(folio))
3537                         return ERR_PTR(-ECHILD);
3538                 if (PageHWPoison(folio_page(folio, 0)) ||
3539                     !folio_test_uptodate(folio)) {
3540                         folio_put(folio);
3541                         return ERR_PTR(-ECHILD);
3542                 }
3543         } else {
3544                 error = shmem_get_folio(inode, 0, &folio, SGP_READ);
3545                 if (error)
3546                         return ERR_PTR(error);
3547                 if (!folio)
3548                         return ERR_PTR(-ECHILD);
3549                 if (PageHWPoison(folio_page(folio, 0))) {
3550                         folio_unlock(folio);
3551                         folio_put(folio);
3552                         return ERR_PTR(-ECHILD);
3553                 }
3554                 folio_unlock(folio);
3555         }
3556         set_delayed_call(done, shmem_put_link, folio);
3557         return folio_address(folio);
3558 }
3559
3560 #ifdef CONFIG_TMPFS_XATTR
3561
3562 static int shmem_fileattr_get(struct dentry *dentry, struct fileattr *fa)
3563 {
3564         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3565
3566         fileattr_fill_flags(fa, info->fsflags & SHMEM_FL_USER_VISIBLE);
3567
3568         return 0;
3569 }
3570
3571 static int shmem_fileattr_set(struct mnt_idmap *idmap,
3572                               struct dentry *dentry, struct fileattr *fa)
3573 {
3574         struct inode *inode = d_inode(dentry);
3575         struct shmem_inode_info *info = SHMEM_I(inode);
3576
3577         if (fileattr_has_fsx(fa))
3578                 return -EOPNOTSUPP;
3579         if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE)
3580                 return -EOPNOTSUPP;
3581
3582         info->fsflags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) |
3583                 (fa->flags & SHMEM_FL_USER_MODIFIABLE);
3584
3585         shmem_set_inode_flags(inode, info->fsflags);
3586         inode_set_ctime_current(inode);
3587         inode_inc_iversion(inode);
3588         return 0;
3589 }
3590
3591 /*
3592  * Superblocks without xattr inode operations may get some security.* xattr
3593  * support from the LSM "for free". As soon as we have any other xattrs
3594  * like ACLs, we also need to implement the security.* handlers at
3595  * filesystem level, though.
3596  */
3597
3598 /*
3599  * Callback for security_inode_init_security() for acquiring xattrs.
3600  */
3601 static int shmem_initxattrs(struct inode *inode,
3602                             const struct xattr *xattr_array, void *fs_info)
3603 {
3604         struct shmem_inode_info *info = SHMEM_I(inode);
3605         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3606         const struct xattr *xattr;
3607         struct simple_xattr *new_xattr;
3608         size_t ispace = 0;
3609         size_t len;
3610
3611         if (sbinfo->max_inodes) {
3612                 for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3613                         ispace += simple_xattr_space(xattr->name,
3614                                 xattr->value_len + XATTR_SECURITY_PREFIX_LEN);
3615                 }
3616                 if (ispace) {
3617                         raw_spin_lock(&sbinfo->stat_lock);
3618                         if (sbinfo->free_ispace < ispace)
3619                                 ispace = 0;
3620                         else
3621                                 sbinfo->free_ispace -= ispace;
3622                         raw_spin_unlock(&sbinfo->stat_lock);
3623                         if (!ispace)
3624                                 return -ENOSPC;
3625                 }
3626         }
3627
3628         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3629                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3630                 if (!new_xattr)
3631                         break;
3632
3633                 len = strlen(xattr->name) + 1;
3634                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3635                                           GFP_KERNEL_ACCOUNT);
3636                 if (!new_xattr->name) {
3637                         kvfree(new_xattr);
3638                         break;
3639                 }
3640
3641                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3642                        XATTR_SECURITY_PREFIX_LEN);
3643                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3644                        xattr->name, len);
3645
3646                 simple_xattr_add(&info->xattrs, new_xattr);
3647         }
3648
3649         if (xattr->name != NULL) {
3650                 if (ispace) {
3651                         raw_spin_lock(&sbinfo->stat_lock);
3652                         sbinfo->free_ispace += ispace;
3653                         raw_spin_unlock(&sbinfo->stat_lock);
3654                 }
3655                 simple_xattrs_free(&info->xattrs, NULL);
3656                 return -ENOMEM;
3657         }
3658
3659         return 0;
3660 }
3661
3662 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3663                                    struct dentry *unused, struct inode *inode,
3664                                    const char *name, void *buffer, size_t size)
3665 {
3666         struct shmem_inode_info *info = SHMEM_I(inode);
3667
3668         name = xattr_full_name(handler, name);
3669         return simple_xattr_get(&info->xattrs, name, buffer, size);
3670 }
3671
3672 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3673                                    struct mnt_idmap *idmap,
3674                                    struct dentry *unused, struct inode *inode,
3675                                    const char *name, const void *value,
3676                                    size_t size, int flags)
3677 {
3678         struct shmem_inode_info *info = SHMEM_I(inode);
3679         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3680         struct simple_xattr *old_xattr;
3681         size_t ispace = 0;
3682
3683         name = xattr_full_name(handler, name);
3684         if (value && sbinfo->max_inodes) {
3685                 ispace = simple_xattr_space(name, size);
3686                 raw_spin_lock(&sbinfo->stat_lock);
3687                 if (sbinfo->free_ispace < ispace)
3688                         ispace = 0;
3689                 else
3690                         sbinfo->free_ispace -= ispace;
3691                 raw_spin_unlock(&sbinfo->stat_lock);
3692                 if (!ispace)
3693                         return -ENOSPC;
3694         }
3695
3696         old_xattr = simple_xattr_set(&info->xattrs, name, value, size, flags);
3697         if (!IS_ERR(old_xattr)) {
3698                 ispace = 0;
3699                 if (old_xattr && sbinfo->max_inodes)
3700                         ispace = simple_xattr_space(old_xattr->name,
3701                                                     old_xattr->size);
3702                 simple_xattr_free(old_xattr);
3703                 old_xattr = NULL;
3704                 inode_set_ctime_current(inode);
3705                 inode_inc_iversion(inode);
3706         }
3707         if (ispace) {
3708                 raw_spin_lock(&sbinfo->stat_lock);
3709                 sbinfo->free_ispace += ispace;
3710                 raw_spin_unlock(&sbinfo->stat_lock);
3711         }
3712         return PTR_ERR(old_xattr);
3713 }
3714
3715 static const struct xattr_handler shmem_security_xattr_handler = {
3716         .prefix = XATTR_SECURITY_PREFIX,
3717         .get = shmem_xattr_handler_get,
3718         .set = shmem_xattr_handler_set,
3719 };
3720
3721 static const struct xattr_handler shmem_trusted_xattr_handler = {
3722         .prefix = XATTR_TRUSTED_PREFIX,
3723         .get = shmem_xattr_handler_get,
3724         .set = shmem_xattr_handler_set,
3725 };
3726
3727 static const struct xattr_handler shmem_user_xattr_handler = {
3728         .prefix = XATTR_USER_PREFIX,
3729         .get = shmem_xattr_handler_get,
3730         .set = shmem_xattr_handler_set,
3731 };
3732
3733 static const struct xattr_handler * const shmem_xattr_handlers[] = {
3734         &shmem_security_xattr_handler,
3735         &shmem_trusted_xattr_handler,
3736         &shmem_user_xattr_handler,
3737         NULL
3738 };
3739
3740 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3741 {
3742         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3743         return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3744 }
3745 #endif /* CONFIG_TMPFS_XATTR */
3746
3747 static const struct inode_operations shmem_short_symlink_operations = {
3748         .getattr        = shmem_getattr,
3749         .setattr        = shmem_setattr,
3750         .get_link       = simple_get_link,
3751 #ifdef CONFIG_TMPFS_XATTR
3752         .listxattr      = shmem_listxattr,
3753 #endif
3754 };
3755
3756 static const struct inode_operations shmem_symlink_inode_operations = {
3757         .getattr        = shmem_getattr,
3758         .setattr        = shmem_setattr,
3759         .get_link       = shmem_get_link,
3760 #ifdef CONFIG_TMPFS_XATTR
3761         .listxattr      = shmem_listxattr,
3762 #endif
3763 };
3764
3765 static struct dentry *shmem_get_parent(struct dentry *child)
3766 {
3767         return ERR_PTR(-ESTALE);
3768 }
3769
3770 static int shmem_match(struct inode *ino, void *vfh)
3771 {
3772         __u32 *fh = vfh;
3773         __u64 inum = fh[2];
3774         inum = (inum << 32) | fh[1];
3775         return ino->i_ino == inum && fh[0] == ino->i_generation;
3776 }
3777
3778 /* Find any alias of inode, but prefer a hashed alias */
3779 static struct dentry *shmem_find_alias(struct inode *inode)
3780 {
3781         struct dentry *alias = d_find_alias(inode);
3782
3783         return alias ?: d_find_any_alias(inode);
3784 }
3785
3786 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3787                 struct fid *fid, int fh_len, int fh_type)
3788 {
3789         struct inode *inode;
3790         struct dentry *dentry = NULL;
3791         u64 inum;
3792
3793         if (fh_len < 3)
3794                 return NULL;
3795
3796         inum = fid->raw[2];
3797         inum = (inum << 32) | fid->raw[1];
3798
3799         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3800                         shmem_match, fid->raw);
3801         if (inode) {
3802                 dentry = shmem_find_alias(inode);
3803                 iput(inode);
3804         }
3805
3806         return dentry;
3807 }
3808
3809 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3810                                 struct inode *parent)
3811 {
3812         if (*len < 3) {
3813                 *len = 3;
3814                 return FILEID_INVALID;
3815         }
3816
3817         if (inode_unhashed(inode)) {
3818                 /* Unfortunately insert_inode_hash is not idempotent,
3819                  * so as we hash inodes here rather than at creation
3820                  * time, we need a lock to ensure we only try
3821                  * to do it once
3822                  */
3823                 static DEFINE_SPINLOCK(lock);
3824                 spin_lock(&lock);
3825                 if (inode_unhashed(inode))
3826                         __insert_inode_hash(inode,
3827                                             inode->i_ino + inode->i_generation);
3828                 spin_unlock(&lock);
3829         }
3830
3831         fh[0] = inode->i_generation;
3832         fh[1] = inode->i_ino;
3833         fh[2] = ((__u64)inode->i_ino) >> 32;
3834
3835         *len = 3;
3836         return 1;
3837 }
3838
3839 static const struct export_operations shmem_export_ops = {
3840         .get_parent     = shmem_get_parent,
3841         .encode_fh      = shmem_encode_fh,
3842         .fh_to_dentry   = shmem_fh_to_dentry,
3843 };
3844
3845 enum shmem_param {
3846         Opt_gid,
3847         Opt_huge,
3848         Opt_mode,
3849         Opt_mpol,
3850         Opt_nr_blocks,
3851         Opt_nr_inodes,
3852         Opt_size,
3853         Opt_uid,
3854         Opt_inode32,
3855         Opt_inode64,
3856         Opt_noswap,
3857         Opt_quota,
3858         Opt_usrquota,
3859         Opt_grpquota,
3860         Opt_usrquota_block_hardlimit,
3861         Opt_usrquota_inode_hardlimit,
3862         Opt_grpquota_block_hardlimit,
3863         Opt_grpquota_inode_hardlimit,
3864 };
3865
3866 static const struct constant_table shmem_param_enums_huge[] = {
3867         {"never",       SHMEM_HUGE_NEVER },
3868         {"always",      SHMEM_HUGE_ALWAYS },
3869         {"within_size", SHMEM_HUGE_WITHIN_SIZE },
3870         {"advise",      SHMEM_HUGE_ADVISE },
3871         {}
3872 };
3873
3874 const struct fs_parameter_spec shmem_fs_parameters[] = {
3875         fsparam_u32   ("gid",           Opt_gid),
3876         fsparam_enum  ("huge",          Opt_huge,  shmem_param_enums_huge),
3877         fsparam_u32oct("mode",          Opt_mode),
3878         fsparam_string("mpol",          Opt_mpol),
3879         fsparam_string("nr_blocks",     Opt_nr_blocks),
3880         fsparam_string("nr_inodes",     Opt_nr_inodes),
3881         fsparam_string("size",          Opt_size),
3882         fsparam_u32   ("uid",           Opt_uid),
3883         fsparam_flag  ("inode32",       Opt_inode32),
3884         fsparam_flag  ("inode64",       Opt_inode64),
3885         fsparam_flag  ("noswap",        Opt_noswap),
3886 #ifdef CONFIG_TMPFS_QUOTA
3887         fsparam_flag  ("quota",         Opt_quota),
3888         fsparam_flag  ("usrquota",      Opt_usrquota),
3889         fsparam_flag  ("grpquota",      Opt_grpquota),
3890         fsparam_string("usrquota_block_hardlimit", Opt_usrquota_block_hardlimit),
3891         fsparam_string("usrquota_inode_hardlimit", Opt_usrquota_inode_hardlimit),
3892         fsparam_string("grpquota_block_hardlimit", Opt_grpquota_block_hardlimit),
3893         fsparam_string("grpquota_inode_hardlimit", Opt_grpquota_inode_hardlimit),
3894 #endif
3895         {}
3896 };
3897
3898 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
3899 {
3900         struct shmem_options *ctx = fc->fs_private;
3901         struct fs_parse_result result;
3902         unsigned long long size;
3903         char *rest;
3904         int opt;
3905         kuid_t kuid;
3906         kgid_t kgid;
3907
3908         opt = fs_parse(fc, shmem_fs_parameters, param, &result);
3909         if (opt < 0)
3910                 return opt;
3911
3912         switch (opt) {
3913         case Opt_size:
3914                 size = memparse(param->string, &rest);
3915                 if (*rest == '%') {
3916                         size <<= PAGE_SHIFT;
3917                         size *= totalram_pages();
3918                         do_div(size, 100);
3919                         rest++;
3920                 }
3921                 if (*rest)
3922                         goto bad_value;
3923                 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
3924                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3925                 break;
3926         case Opt_nr_blocks:
3927                 ctx->blocks = memparse(param->string, &rest);
3928                 if (*rest || ctx->blocks > LONG_MAX)
3929                         goto bad_value;
3930                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3931                 break;
3932         case Opt_nr_inodes:
3933                 ctx->inodes = memparse(param->string, &rest);
3934                 if (*rest || ctx->inodes > ULONG_MAX / BOGO_INODE_SIZE)
3935                         goto bad_value;
3936                 ctx->seen |= SHMEM_SEEN_INODES;
3937                 break;
3938         case Opt_mode:
3939                 ctx->mode = result.uint_32 & 07777;
3940                 break;
3941         case Opt_uid:
3942                 kuid = make_kuid(current_user_ns(), result.uint_32);
3943                 if (!uid_valid(kuid))
3944                         goto bad_value;
3945
3946                 /*
3947                  * The requested uid must be representable in the
3948                  * filesystem's idmapping.
3949                  */
3950                 if (!kuid_has_mapping(fc->user_ns, kuid))
3951                         goto bad_value;
3952
3953                 ctx->uid = kuid;
3954                 break;
3955         case Opt_gid:
3956                 kgid = make_kgid(current_user_ns(), result.uint_32);
3957                 if (!gid_valid(kgid))
3958                         goto bad_value;
3959
3960                 /*
3961                  * The requested gid must be representable in the
3962                  * filesystem's idmapping.
3963                  */
3964                 if (!kgid_has_mapping(fc->user_ns, kgid))
3965                         goto bad_value;
3966
3967                 ctx->gid = kgid;
3968                 break;
3969         case Opt_huge:
3970                 ctx->huge = result.uint_32;
3971                 if (ctx->huge != SHMEM_HUGE_NEVER &&
3972                     !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
3973                       has_transparent_hugepage()))
3974                         goto unsupported_parameter;
3975                 ctx->seen |= SHMEM_SEEN_HUGE;
3976                 break;
3977         case Opt_mpol:
3978                 if (IS_ENABLED(CONFIG_NUMA)) {
3979                         mpol_put(ctx->mpol);
3980                         ctx->mpol = NULL;
3981                         if (mpol_parse_str(param->string, &ctx->mpol))
3982                                 goto bad_value;
3983                         break;
3984                 }
3985                 goto unsupported_parameter;
3986         case Opt_inode32:
3987                 ctx->full_inums = false;
3988                 ctx->seen |= SHMEM_SEEN_INUMS;
3989                 break;
3990         case Opt_inode64:
3991                 if (sizeof(ino_t) < 8) {
3992                         return invalfc(fc,
3993                                        "Cannot use inode64 with <64bit inums in kernel\n");
3994                 }
3995                 ctx->full_inums = true;
3996                 ctx->seen |= SHMEM_SEEN_INUMS;
3997                 break;
3998         case Opt_noswap:
3999                 if ((fc->user_ns != &init_user_ns) || !capable(CAP_SYS_ADMIN)) {
4000                         return invalfc(fc,
4001                                        "Turning off swap in unprivileged tmpfs mounts unsupported");
4002                 }
4003                 ctx->noswap = true;
4004                 ctx->seen |= SHMEM_SEEN_NOSWAP;
4005                 break;
4006         case Opt_quota:
4007                 if (fc->user_ns != &init_user_ns)
4008                         return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4009                 ctx->seen |= SHMEM_SEEN_QUOTA;
4010                 ctx->quota_types |= (QTYPE_MASK_USR | QTYPE_MASK_GRP);
4011                 break;
4012         case Opt_usrquota:
4013                 if (fc->user_ns != &init_user_ns)
4014                         return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4015                 ctx->seen |= SHMEM_SEEN_QUOTA;
4016                 ctx->quota_types |= QTYPE_MASK_USR;
4017                 break;
4018         case Opt_grpquota:
4019                 if (fc->user_ns != &init_user_ns)
4020                         return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported");
4021                 ctx->seen |= SHMEM_SEEN_QUOTA;
4022                 ctx->quota_types |= QTYPE_MASK_GRP;
4023                 break;
4024         case Opt_usrquota_block_hardlimit:
4025                 size = memparse(param->string, &rest);
4026                 if (*rest || !size)
4027                         goto bad_value;
4028                 if (size > SHMEM_QUOTA_MAX_SPC_LIMIT)
4029                         return invalfc(fc,
4030                                        "User quota block hardlimit too large.");
4031                 ctx->qlimits.usrquota_bhardlimit = size;
4032                 break;
4033         case Opt_grpquota_block_hardlimit:
4034                 size = memparse(param->string, &rest);
4035                 if (*rest || !size)
4036                         goto bad_value;
4037                 if (size > SHMEM_QUOTA_MAX_SPC_LIMIT)
4038                         return invalfc(fc,
4039                                        "Group quota block hardlimit too large.");
4040                 ctx->qlimits.grpquota_bhardlimit = size;
4041                 break;
4042         case Opt_usrquota_inode_hardlimit:
4043                 size = memparse(param->string, &rest);
4044                 if (*rest || !size)
4045                         goto bad_value;
4046                 if (size > SHMEM_QUOTA_MAX_INO_LIMIT)
4047                         return invalfc(fc,
4048                                        "User quota inode hardlimit too large.");
4049                 ctx->qlimits.usrquota_ihardlimit = size;
4050                 break;
4051         case Opt_grpquota_inode_hardlimit:
4052                 size = memparse(param->string, &rest);
4053                 if (*rest || !size)
4054                         goto bad_value;
4055                 if (size > SHMEM_QUOTA_MAX_INO_LIMIT)
4056                         return invalfc(fc,
4057                                        "Group quota inode hardlimit too large.");
4058                 ctx->qlimits.grpquota_ihardlimit = size;
4059                 break;
4060         }
4061         return 0;
4062
4063 unsupported_parameter:
4064         return invalfc(fc, "Unsupported parameter '%s'", param->key);
4065 bad_value:
4066         return invalfc(fc, "Bad value for '%s'", param->key);
4067 }
4068
4069 static int shmem_parse_options(struct fs_context *fc, void *data)
4070 {
4071         char *options = data;
4072
4073         if (options) {
4074                 int err = security_sb_eat_lsm_opts(options, &fc->security);
4075                 if (err)
4076                         return err;
4077         }
4078
4079         while (options != NULL) {
4080                 char *this_char = options;
4081                 for (;;) {
4082                         /*
4083                          * NUL-terminate this option: unfortunately,
4084                          * mount options form a comma-separated list,
4085                          * but mpol's nodelist may also contain commas.
4086                          */
4087                         options = strchr(options, ',');
4088                         if (options == NULL)
4089                                 break;
4090                         options++;
4091                         if (!isdigit(*options)) {
4092                                 options[-1] = '\0';
4093                                 break;
4094                         }
4095                 }
4096                 if (*this_char) {
4097                         char *value = strchr(this_char, '=');
4098                         size_t len = 0;
4099                         int err;
4100
4101                         if (value) {
4102                                 *value++ = '\0';
4103                                 len = strlen(value);
4104                         }
4105                         err = vfs_parse_fs_string(fc, this_char, value, len);
4106                         if (err < 0)
4107                                 return err;
4108                 }
4109         }
4110         return 0;
4111 }
4112
4113 /*
4114  * Reconfigure a shmem filesystem.
4115  */
4116 static int shmem_reconfigure(struct fs_context *fc)
4117 {
4118         struct shmem_options *ctx = fc->fs_private;
4119         struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
4120         unsigned long used_isp;
4121         struct mempolicy *mpol = NULL;
4122         const char *err;
4123
4124         raw_spin_lock(&sbinfo->stat_lock);
4125         used_isp = sbinfo->max_inodes * BOGO_INODE_SIZE - sbinfo->free_ispace;
4126
4127         if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
4128                 if (!sbinfo->max_blocks) {
4129                         err = "Cannot retroactively limit size";
4130                         goto out;
4131                 }
4132                 if (percpu_counter_compare(&sbinfo->used_blocks,
4133                                            ctx->blocks) > 0) {
4134                         err = "Too small a size for current use";
4135                         goto out;
4136                 }
4137         }
4138         if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
4139                 if (!sbinfo->max_inodes) {
4140                         err = "Cannot retroactively limit inodes";
4141                         goto out;
4142                 }
4143                 if (ctx->inodes * BOGO_INODE_SIZE < used_isp) {
4144                         err = "Too few inodes for current use";
4145                         goto out;
4146                 }
4147         }
4148
4149         if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums &&
4150             sbinfo->next_ino > UINT_MAX) {
4151                 err = "Current inum too high to switch to 32-bit inums";
4152                 goto out;
4153         }
4154         if ((ctx->seen & SHMEM_SEEN_NOSWAP) && ctx->noswap && !sbinfo->noswap) {
4155                 err = "Cannot disable swap on remount";
4156                 goto out;
4157         }
4158         if (!(ctx->seen & SHMEM_SEEN_NOSWAP) && !ctx->noswap && sbinfo->noswap) {
4159                 err = "Cannot enable swap on remount if it was disabled on first mount";
4160                 goto out;
4161         }
4162
4163         if (ctx->seen & SHMEM_SEEN_QUOTA &&
4164             !sb_any_quota_loaded(fc->root->d_sb)) {
4165                 err = "Cannot enable quota on remount";
4166                 goto out;
4167         }
4168
4169 #ifdef CONFIG_TMPFS_QUOTA
4170 #define CHANGED_LIMIT(name)                                             \
4171         (ctx->qlimits.name## hardlimit &&                               \
4172         (ctx->qlimits.name## hardlimit != sbinfo->qlimits.name## hardlimit))
4173
4174         if (CHANGED_LIMIT(usrquota_b) || CHANGED_LIMIT(usrquota_i) ||
4175             CHANGED_LIMIT(grpquota_b) || CHANGED_LIMIT(grpquota_i)) {
4176                 err = "Cannot change global quota limit on remount";
4177                 goto out;
4178         }
4179 #endif /* CONFIG_TMPFS_QUOTA */
4180
4181         if (ctx->seen & SHMEM_SEEN_HUGE)
4182                 sbinfo->huge = ctx->huge;
4183         if (ctx->seen & SHMEM_SEEN_INUMS)
4184                 sbinfo->full_inums = ctx->full_inums;
4185         if (ctx->seen & SHMEM_SEEN_BLOCKS)
4186                 sbinfo->max_blocks  = ctx->blocks;
4187         if (ctx->seen & SHMEM_SEEN_INODES) {
4188                 sbinfo->max_inodes  = ctx->inodes;
4189                 sbinfo->free_ispace = ctx->inodes * BOGO_INODE_SIZE - used_isp;
4190         }
4191
4192         /*
4193          * Preserve previous mempolicy unless mpol remount option was specified.
4194          */
4195         if (ctx->mpol) {
4196                 mpol = sbinfo->mpol;
4197                 sbinfo->mpol = ctx->mpol;       /* transfers initial ref */
4198                 ctx->mpol = NULL;
4199         }
4200
4201         if (ctx->noswap)
4202                 sbinfo->noswap = true;
4203
4204         raw_spin_unlock(&sbinfo->stat_lock);
4205         mpol_put(mpol);
4206         return 0;
4207 out:
4208         raw_spin_unlock(&sbinfo->stat_lock);
4209         return invalfc(fc, "%s", err);
4210 }
4211
4212 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
4213 {
4214         struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
4215         struct mempolicy *mpol;
4216
4217         if (sbinfo->max_blocks != shmem_default_max_blocks())
4218                 seq_printf(seq, ",size=%luk", K(sbinfo->max_blocks));
4219         if (sbinfo->max_inodes != shmem_default_max_inodes())
4220                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
4221         if (sbinfo->mode != (0777 | S_ISVTX))
4222                 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
4223         if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
4224                 seq_printf(seq, ",uid=%u",
4225                                 from_kuid_munged(&init_user_ns, sbinfo->uid));
4226         if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
4227                 seq_printf(seq, ",gid=%u",
4228                                 from_kgid_munged(&init_user_ns, sbinfo->gid));
4229
4230         /*
4231          * Showing inode{64,32} might be useful even if it's the system default,
4232          * since then people don't have to resort to checking both here and
4233          * /proc/config.gz to confirm 64-bit inums were successfully applied
4234          * (which may not even exist if IKCONFIG_PROC isn't enabled).
4235          *
4236          * We hide it when inode64 isn't the default and we are using 32-bit
4237          * inodes, since that probably just means the feature isn't even under
4238          * consideration.
4239          *
4240          * As such:
4241          *
4242          *                     +-----------------+-----------------+
4243          *                     | TMPFS_INODE64=y | TMPFS_INODE64=n |
4244          *  +------------------+-----------------+-----------------+
4245          *  | full_inums=true  | show            | show            |
4246          *  | full_inums=false | show            | hide            |
4247          *  +------------------+-----------------+-----------------+
4248          *
4249          */
4250         if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums)
4251                 seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32));
4252 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4253         /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
4254         if (sbinfo->huge)
4255                 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
4256 #endif
4257         mpol = shmem_get_sbmpol(sbinfo);
4258         shmem_show_mpol(seq, mpol);
4259         mpol_put(mpol);
4260         if (sbinfo->noswap)
4261                 seq_printf(seq, ",noswap");
4262         return 0;
4263 }
4264
4265 #endif /* CONFIG_TMPFS */
4266
4267 static void shmem_put_super(struct super_block *sb)
4268 {
4269         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
4270
4271 #ifdef CONFIG_TMPFS_QUOTA
4272         shmem_disable_quotas(sb);
4273 #endif
4274         free_percpu(sbinfo->ino_batch);
4275         percpu_counter_destroy(&sbinfo->used_blocks);
4276         mpol_put(sbinfo->mpol);
4277         kfree(sbinfo);
4278         sb->s_fs_info = NULL;
4279 }
4280
4281 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
4282 {
4283         struct shmem_options *ctx = fc->fs_private;
4284         struct inode *inode;
4285         struct shmem_sb_info *sbinfo;
4286         int error = -ENOMEM;
4287
4288         /* Round up to L1_CACHE_BYTES to resist false sharing */
4289         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
4290                                 L1_CACHE_BYTES), GFP_KERNEL);
4291         if (!sbinfo)
4292                 return error;
4293
4294         sb->s_fs_info = sbinfo;
4295
4296 #ifdef CONFIG_TMPFS
4297         /*
4298          * Per default we only allow half of the physical ram per
4299          * tmpfs instance, limiting inodes to one per page of lowmem;
4300          * but the internal instance is left unlimited.
4301          */
4302         if (!(sb->s_flags & SB_KERNMOUNT)) {
4303                 if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
4304                         ctx->blocks = shmem_default_max_blocks();
4305                 if (!(ctx->seen & SHMEM_SEEN_INODES))
4306                         ctx->inodes = shmem_default_max_inodes();
4307                 if (!(ctx->seen & SHMEM_SEEN_INUMS))
4308                         ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64);
4309                 sbinfo->noswap = ctx->noswap;
4310         } else {
4311                 sb->s_flags |= SB_NOUSER;
4312         }
4313         sb->s_export_op = &shmem_export_ops;
4314         sb->s_flags |= SB_NOSEC | SB_I_VERSION;
4315 #else
4316         sb->s_flags |= SB_NOUSER;
4317 #endif
4318         sbinfo->max_blocks = ctx->blocks;
4319         sbinfo->max_inodes = ctx->inodes;
4320         sbinfo->free_ispace = sbinfo->max_inodes * BOGO_INODE_SIZE;
4321         if (sb->s_flags & SB_KERNMOUNT) {
4322                 sbinfo->ino_batch = alloc_percpu(ino_t);
4323                 if (!sbinfo->ino_batch)
4324                         goto failed;
4325         }
4326         sbinfo->uid = ctx->uid;
4327         sbinfo->gid = ctx->gid;
4328         sbinfo->full_inums = ctx->full_inums;
4329         sbinfo->mode = ctx->mode;
4330         sbinfo->huge = ctx->huge;
4331         sbinfo->mpol = ctx->mpol;
4332         ctx->mpol = NULL;
4333
4334         raw_spin_lock_init(&sbinfo->stat_lock);
4335         if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
4336                 goto failed;
4337         spin_lock_init(&sbinfo->shrinklist_lock);
4338         INIT_LIST_HEAD(&sbinfo->shrinklist);
4339
4340         sb->s_maxbytes = MAX_LFS_FILESIZE;
4341         sb->s_blocksize = PAGE_SIZE;
4342         sb->s_blocksize_bits = PAGE_SHIFT;
4343         sb->s_magic = TMPFS_MAGIC;
4344         sb->s_op = &shmem_ops;
4345         sb->s_time_gran = 1;
4346 #ifdef CONFIG_TMPFS_XATTR
4347         sb->s_xattr = shmem_xattr_handlers;
4348 #endif
4349 #ifdef CONFIG_TMPFS_POSIX_ACL
4350         sb->s_flags |= SB_POSIXACL;
4351 #endif
4352         uuid_gen(&sb->s_uuid);
4353
4354 #ifdef CONFIG_TMPFS_QUOTA
4355         if (ctx->seen & SHMEM_SEEN_QUOTA) {
4356                 sb->dq_op = &shmem_quota_operations;
4357                 sb->s_qcop = &dquot_quotactl_sysfile_ops;
4358                 sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP;
4359
4360                 /* Copy the default limits from ctx into sbinfo */
4361                 memcpy(&sbinfo->qlimits, &ctx->qlimits,
4362                        sizeof(struct shmem_quota_limits));
4363
4364                 if (shmem_enable_quotas(sb, ctx->quota_types))
4365                         goto failed;
4366         }
4367 #endif /* CONFIG_TMPFS_QUOTA */
4368
4369         inode = shmem_get_inode(&nop_mnt_idmap, sb, NULL,
4370                                 S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
4371         if (IS_ERR(inode)) {
4372                 error = PTR_ERR(inode);
4373                 goto failed;
4374         }
4375         inode->i_uid = sbinfo->uid;
4376         inode->i_gid = sbinfo->gid;
4377         sb->s_root = d_make_root(inode);
4378         if (!sb->s_root)
4379                 goto failed;
4380         return 0;
4381
4382 failed:
4383         shmem_put_super(sb);
4384         return error;
4385 }
4386
4387 static int shmem_get_tree(struct fs_context *fc)
4388 {
4389         return get_tree_nodev(fc, shmem_fill_super);
4390 }
4391
4392 static void shmem_free_fc(struct fs_context *fc)
4393 {
4394         struct shmem_options *ctx = fc->fs_private;
4395
4396         if (ctx) {
4397                 mpol_put(ctx->mpol);
4398                 kfree(ctx);
4399         }
4400 }
4401
4402 static const struct fs_context_operations shmem_fs_context_ops = {
4403         .free                   = shmem_free_fc,
4404         .get_tree               = shmem_get_tree,
4405 #ifdef CONFIG_TMPFS
4406         .parse_monolithic       = shmem_parse_options,
4407         .parse_param            = shmem_parse_one,
4408         .reconfigure            = shmem_reconfigure,
4409 #endif
4410 };
4411
4412 static struct kmem_cache *shmem_inode_cachep __ro_after_init;
4413
4414 static struct inode *shmem_alloc_inode(struct super_block *sb)
4415 {
4416         struct shmem_inode_info *info;
4417         info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL);
4418         if (!info)
4419                 return NULL;
4420         return &info->vfs_inode;
4421 }
4422
4423 static void shmem_free_in_core_inode(struct inode *inode)
4424 {
4425         if (S_ISLNK(inode->i_mode))
4426                 kfree(inode->i_link);
4427         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
4428 }
4429
4430 static void shmem_destroy_inode(struct inode *inode)
4431 {
4432         if (S_ISREG(inode->i_mode))
4433                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
4434         if (S_ISDIR(inode->i_mode))
4435                 simple_offset_destroy(shmem_get_offset_ctx(inode));
4436 }
4437
4438 static void shmem_init_inode(void *foo)
4439 {
4440         struct shmem_inode_info *info = foo;
4441         inode_init_once(&info->vfs_inode);
4442 }
4443
4444 static void __init shmem_init_inodecache(void)
4445 {
4446         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
4447                                 sizeof(struct shmem_inode_info),
4448                                 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
4449 }
4450
4451 static void __init shmem_destroy_inodecache(void)
4452 {
4453         kmem_cache_destroy(shmem_inode_cachep);
4454 }
4455
4456 /* Keep the page in page cache instead of truncating it */
4457 static int shmem_error_remove_folio(struct address_space *mapping,
4458                                    struct folio *folio)
4459 {
4460         return 0;
4461 }
4462
4463 const struct address_space_operations shmem_aops = {
4464         .writepage      = shmem_writepage,
4465         .dirty_folio    = noop_dirty_folio,
4466 #ifdef CONFIG_TMPFS
4467         .write_begin    = shmem_write_begin,
4468         .write_end      = shmem_write_end,
4469 #endif
4470 #ifdef CONFIG_MIGRATION
4471         .migrate_folio  = migrate_folio,
4472 #endif
4473         .error_remove_folio = shmem_error_remove_folio,
4474 };
4475 EXPORT_SYMBOL(shmem_aops);
4476
4477 static const struct file_operations shmem_file_operations = {
4478         .mmap           = shmem_mmap,
4479         .open           = shmem_file_open,
4480         .get_unmapped_area = shmem_get_unmapped_area,
4481 #ifdef CONFIG_TMPFS
4482         .llseek         = shmem_file_llseek,
4483         .read_iter      = shmem_file_read_iter,
4484         .write_iter     = shmem_file_write_iter,
4485         .fsync          = noop_fsync,
4486         .splice_read    = shmem_file_splice_read,
4487         .splice_write   = iter_file_splice_write,
4488         .fallocate      = shmem_fallocate,
4489 #endif
4490 };
4491
4492 static const struct inode_operations shmem_inode_operations = {
4493         .getattr        = shmem_getattr,
4494         .setattr        = shmem_setattr,
4495 #ifdef CONFIG_TMPFS_XATTR
4496         .listxattr      = shmem_listxattr,
4497         .set_acl        = simple_set_acl,
4498         .fileattr_get   = shmem_fileattr_get,
4499         .fileattr_set   = shmem_fileattr_set,
4500 #endif
4501 };
4502
4503 static const struct inode_operations shmem_dir_inode_operations = {
4504 #ifdef CONFIG_TMPFS
4505         .getattr        = shmem_getattr,
4506         .create         = shmem_create,
4507         .lookup         = simple_lookup,
4508         .link           = shmem_link,
4509         .unlink         = shmem_unlink,
4510         .symlink        = shmem_symlink,
4511         .mkdir          = shmem_mkdir,
4512         .rmdir          = shmem_rmdir,
4513         .mknod          = shmem_mknod,
4514         .rename         = shmem_rename2,
4515         .tmpfile        = shmem_tmpfile,
4516         .get_offset_ctx = shmem_get_offset_ctx,
4517 #endif
4518 #ifdef CONFIG_TMPFS_XATTR
4519         .listxattr      = shmem_listxattr,
4520         .fileattr_get   = shmem_fileattr_get,
4521         .fileattr_set   = shmem_fileattr_set,
4522 #endif
4523 #ifdef CONFIG_TMPFS_POSIX_ACL
4524         .setattr        = shmem_setattr,
4525         .set_acl        = simple_set_acl,
4526 #endif
4527 };
4528
4529 static const struct inode_operations shmem_special_inode_operations = {
4530         .getattr        = shmem_getattr,
4531 #ifdef CONFIG_TMPFS_XATTR
4532         .listxattr      = shmem_listxattr,
4533 #endif
4534 #ifdef CONFIG_TMPFS_POSIX_ACL
4535         .setattr        = shmem_setattr,
4536         .set_acl        = simple_set_acl,
4537 #endif
4538 };
4539
4540 static const struct super_operations shmem_ops = {
4541         .alloc_inode    = shmem_alloc_inode,
4542         .free_inode     = shmem_free_in_core_inode,
4543         .destroy_inode  = shmem_destroy_inode,
4544 #ifdef CONFIG_TMPFS
4545         .statfs         = shmem_statfs,
4546         .show_options   = shmem_show_options,
4547 #endif
4548 #ifdef CONFIG_TMPFS_QUOTA
4549         .get_dquots     = shmem_get_dquots,
4550 #endif
4551         .evict_inode    = shmem_evict_inode,
4552         .drop_inode     = generic_delete_inode,
4553         .put_super      = shmem_put_super,
4554 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4555         .nr_cached_objects      = shmem_unused_huge_count,
4556         .free_cached_objects    = shmem_unused_huge_scan,
4557 #endif
4558 };
4559
4560 static const struct vm_operations_struct shmem_vm_ops = {
4561         .fault          = shmem_fault,
4562         .map_pages      = filemap_map_pages,
4563 #ifdef CONFIG_NUMA
4564         .set_policy     = shmem_set_policy,
4565         .get_policy     = shmem_get_policy,
4566 #endif
4567 };
4568
4569 static const struct vm_operations_struct shmem_anon_vm_ops = {
4570         .fault          = shmem_fault,
4571         .map_pages      = filemap_map_pages,
4572 #ifdef CONFIG_NUMA
4573         .set_policy     = shmem_set_policy,
4574         .get_policy     = shmem_get_policy,
4575 #endif
4576 };
4577
4578 int shmem_init_fs_context(struct fs_context *fc)
4579 {
4580         struct shmem_options *ctx;
4581
4582         ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
4583         if (!ctx)
4584                 return -ENOMEM;
4585
4586         ctx->mode = 0777 | S_ISVTX;
4587         ctx->uid = current_fsuid();
4588         ctx->gid = current_fsgid();
4589
4590         fc->fs_private = ctx;
4591         fc->ops = &shmem_fs_context_ops;
4592         return 0;
4593 }
4594
4595 static struct file_system_type shmem_fs_type = {
4596         .owner          = THIS_MODULE,
4597         .name           = "tmpfs",
4598         .init_fs_context = shmem_init_fs_context,
4599 #ifdef CONFIG_TMPFS
4600         .parameters     = shmem_fs_parameters,
4601 #endif
4602         .kill_sb        = kill_litter_super,
4603         .fs_flags       = FS_USERNS_MOUNT | FS_ALLOW_IDMAP,
4604 };
4605
4606 void __init shmem_init(void)
4607 {
4608         int error;
4609
4610         shmem_init_inodecache();
4611
4612 #ifdef CONFIG_TMPFS_QUOTA
4613         error = register_quota_format(&shmem_quota_format);
4614         if (error < 0) {
4615                 pr_err("Could not register quota format\n");
4616                 goto out3;
4617         }
4618 #endif
4619
4620         error = register_filesystem(&shmem_fs_type);
4621         if (error) {
4622                 pr_err("Could not register tmpfs\n");
4623                 goto out2;
4624         }
4625
4626         shm_mnt = kern_mount(&shmem_fs_type);
4627         if (IS_ERR(shm_mnt)) {
4628                 error = PTR_ERR(shm_mnt);
4629                 pr_err("Could not kern_mount tmpfs\n");
4630                 goto out1;
4631         }
4632
4633 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4634         if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
4635                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4636         else
4637                 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */
4638 #endif
4639         return;
4640
4641 out1:
4642         unregister_filesystem(&shmem_fs_type);
4643 out2:
4644 #ifdef CONFIG_TMPFS_QUOTA
4645         unregister_quota_format(&shmem_quota_format);
4646 out3:
4647 #endif
4648         shmem_destroy_inodecache();
4649         shm_mnt = ERR_PTR(error);
4650 }
4651
4652 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
4653 static ssize_t shmem_enabled_show(struct kobject *kobj,
4654                                   struct kobj_attribute *attr, char *buf)
4655 {
4656         static const int values[] = {
4657                 SHMEM_HUGE_ALWAYS,
4658                 SHMEM_HUGE_WITHIN_SIZE,
4659                 SHMEM_HUGE_ADVISE,
4660                 SHMEM_HUGE_NEVER,
4661                 SHMEM_HUGE_DENY,
4662                 SHMEM_HUGE_FORCE,
4663         };
4664         int len = 0;
4665         int i;
4666
4667         for (i = 0; i < ARRAY_SIZE(values); i++) {
4668                 len += sysfs_emit_at(buf, len,
4669                                 shmem_huge == values[i] ? "%s[%s]" : "%s%s",
4670                                 i ? " " : "", shmem_format_huge(values[i]));
4671         }
4672         len += sysfs_emit_at(buf, len, "\n");
4673
4674         return len;
4675 }
4676
4677 static ssize_t shmem_enabled_store(struct kobject *kobj,
4678                 struct kobj_attribute *attr, const char *buf, size_t count)
4679 {
4680         char tmp[16];
4681         int huge;
4682
4683         if (count + 1 > sizeof(tmp))
4684                 return -EINVAL;
4685         memcpy(tmp, buf, count);
4686         tmp[count] = '\0';
4687         if (count && tmp[count - 1] == '\n')
4688                 tmp[count - 1] = '\0';
4689
4690         huge = shmem_parse_huge(tmp);
4691         if (huge == -EINVAL)
4692                 return -EINVAL;
4693         if (!has_transparent_hugepage() &&
4694                         huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
4695                 return -EINVAL;
4696
4697         shmem_huge = huge;
4698         if (shmem_huge > SHMEM_HUGE_DENY)
4699                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4700         return count;
4701 }
4702
4703 struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled);
4704 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
4705
4706 #else /* !CONFIG_SHMEM */
4707
4708 /*
4709  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
4710  *
4711  * This is intended for small system where the benefits of the full
4712  * shmem code (swap-backed and resource-limited) are outweighed by
4713  * their complexity. On systems without swap this code should be
4714  * effectively equivalent, but much lighter weight.
4715  */
4716
4717 static struct file_system_type shmem_fs_type = {
4718         .name           = "tmpfs",
4719         .init_fs_context = ramfs_init_fs_context,
4720         .parameters     = ramfs_fs_parameters,
4721         .kill_sb        = ramfs_kill_sb,
4722         .fs_flags       = FS_USERNS_MOUNT,
4723 };
4724
4725 void __init shmem_init(void)
4726 {
4727         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4728
4729         shm_mnt = kern_mount(&shmem_fs_type);
4730         BUG_ON(IS_ERR(shm_mnt));
4731 }
4732
4733 int shmem_unuse(unsigned int type)
4734 {
4735         return 0;
4736 }
4737
4738 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
4739 {
4740         return 0;
4741 }
4742
4743 void shmem_unlock_mapping(struct address_space *mapping)
4744 {
4745 }
4746
4747 #ifdef CONFIG_MMU
4748 unsigned long shmem_get_unmapped_area(struct file *file,
4749                                       unsigned long addr, unsigned long len,
4750                                       unsigned long pgoff, unsigned long flags)
4751 {
4752         return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
4753 }
4754 #endif
4755
4756 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4757 {
4758         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4759 }
4760 EXPORT_SYMBOL_GPL(shmem_truncate_range);
4761
4762 #define shmem_vm_ops                            generic_file_vm_ops
4763 #define shmem_anon_vm_ops                       generic_file_vm_ops
4764 #define shmem_file_operations                   ramfs_file_operations
4765 #define shmem_acct_size(flags, size)            0
4766 #define shmem_unacct_size(flags, size)          do {} while (0)
4767
4768 static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap,
4769                                 struct super_block *sb, struct inode *dir,
4770                                 umode_t mode, dev_t dev, unsigned long flags)
4771 {
4772         struct inode *inode = ramfs_get_inode(sb, dir, mode, dev);
4773         return inode ? inode : ERR_PTR(-ENOSPC);
4774 }
4775
4776 #endif /* CONFIG_SHMEM */
4777
4778 /* common code */
4779
4780 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name,
4781                         loff_t size, unsigned long flags, unsigned int i_flags)
4782 {
4783         struct inode *inode;
4784         struct file *res;
4785
4786         if (IS_ERR(mnt))
4787                 return ERR_CAST(mnt);
4788
4789         if (size < 0 || size > MAX_LFS_FILESIZE)
4790                 return ERR_PTR(-EINVAL);
4791
4792         if (shmem_acct_size(flags, size))
4793                 return ERR_PTR(-ENOMEM);
4794
4795         if (is_idmapped_mnt(mnt))
4796                 return ERR_PTR(-EINVAL);
4797
4798         inode = shmem_get_inode(&nop_mnt_idmap, mnt->mnt_sb, NULL,
4799                                 S_IFREG | S_IRWXUGO, 0, flags);
4800         if (IS_ERR(inode)) {
4801                 shmem_unacct_size(flags, size);
4802                 return ERR_CAST(inode);
4803         }
4804         inode->i_flags |= i_flags;
4805         inode->i_size = size;
4806         clear_nlink(inode);     /* It is unlinked */
4807         res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4808         if (!IS_ERR(res))
4809                 res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
4810                                 &shmem_file_operations);
4811         if (IS_ERR(res))
4812                 iput(inode);
4813         return res;
4814 }
4815
4816 /**
4817  * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
4818  *      kernel internal.  There will be NO LSM permission checks against the
4819  *      underlying inode.  So users of this interface must do LSM checks at a
4820  *      higher layer.  The users are the big_key and shm implementations.  LSM
4821  *      checks are provided at the key or shm level rather than the inode.
4822  * @name: name for dentry (to be seen in /proc/<pid>/maps
4823  * @size: size to be set for the file
4824  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4825  */
4826 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
4827 {
4828         return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
4829 }
4830
4831 /**
4832  * shmem_file_setup - get an unlinked file living in tmpfs
4833  * @name: name for dentry (to be seen in /proc/<pid>/maps
4834  * @size: size to be set for the file
4835  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4836  */
4837 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
4838 {
4839         return __shmem_file_setup(shm_mnt, name, size, flags, 0);
4840 }
4841 EXPORT_SYMBOL_GPL(shmem_file_setup);
4842
4843 /**
4844  * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
4845  * @mnt: the tmpfs mount where the file will be created
4846  * @name: name for dentry (to be seen in /proc/<pid>/maps
4847  * @size: size to be set for the file
4848  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4849  */
4850 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
4851                                        loff_t size, unsigned long flags)
4852 {
4853         return __shmem_file_setup(mnt, name, size, flags, 0);
4854 }
4855 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
4856
4857 /**
4858  * shmem_zero_setup - setup a shared anonymous mapping
4859  * @vma: the vma to be mmapped is prepared by do_mmap
4860  */
4861 int shmem_zero_setup(struct vm_area_struct *vma)
4862 {
4863         struct file *file;
4864         loff_t size = vma->vm_end - vma->vm_start;
4865
4866         /*
4867          * Cloning a new file under mmap_lock leads to a lock ordering conflict
4868          * between XFS directory reading and selinux: since this file is only
4869          * accessible to the user through its mapping, use S_PRIVATE flag to
4870          * bypass file security, in the same way as shmem_kernel_file_setup().
4871          */
4872         file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
4873         if (IS_ERR(file))
4874                 return PTR_ERR(file);
4875
4876         if (vma->vm_file)
4877                 fput(vma->vm_file);
4878         vma->vm_file = file;
4879         vma->vm_ops = &shmem_anon_vm_ops;
4880
4881         return 0;
4882 }
4883
4884 /**
4885  * shmem_read_folio_gfp - read into page cache, using specified page allocation flags.
4886  * @mapping:    the folio's address_space
4887  * @index:      the folio index
4888  * @gfp:        the page allocator flags to use if allocating
4889  *
4890  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
4891  * with any new page allocations done using the specified allocation flags.
4892  * But read_cache_page_gfp() uses the ->read_folio() method: which does not
4893  * suit tmpfs, since it may have pages in swapcache, and needs to find those
4894  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
4895  *
4896  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
4897  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4898  */
4899 struct folio *shmem_read_folio_gfp(struct address_space *mapping,
4900                 pgoff_t index, gfp_t gfp)
4901 {
4902 #ifdef CONFIG_SHMEM
4903         struct inode *inode = mapping->host;
4904         struct folio *folio;
4905         int error;
4906
4907         BUG_ON(!shmem_mapping(mapping));
4908         error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE,
4909                                     gfp, NULL, NULL);
4910         if (error)
4911                 return ERR_PTR(error);
4912
4913         folio_unlock(folio);
4914         return folio;
4915 #else
4916         /*
4917          * The tiny !SHMEM case uses ramfs without swap
4918          */
4919         return mapping_read_folio_gfp(mapping, index, gfp);
4920 #endif
4921 }
4922 EXPORT_SYMBOL_GPL(shmem_read_folio_gfp);
4923
4924 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4925                                          pgoff_t index, gfp_t gfp)
4926 {
4927         struct folio *folio = shmem_read_folio_gfp(mapping, index, gfp);
4928         struct page *page;
4929
4930         if (IS_ERR(folio))
4931                 return &folio->page;
4932
4933         page = folio_file_page(folio, index);
4934         if (PageHWPoison(page)) {
4935                 folio_put(folio);
4936                 return ERR_PTR(-EIO);
4937         }
4938
4939         return page;
4940 }
4941 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);