overlayfs/file.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2017 Red Hat, Inc.
   4  */
   5
   6 #include <linux/cred.h>
   7 #include <linux/file.h>
   8 #include <linux/mount.h>
   9 #include <linux/xattr.h>
  10 #include <linux/uio.h>
  11 #include <linux/uaccess.h>
  12 #include <linux/splice.h>
  13 #include <linux/security.h>
  14 #include <linux/mm.h>
  15 #include <linux/fs.h>
  16 #include "overlayfs.h"
  17
  18 struct ovl_aio_req {
  19         struct kiocb iocb;
  20         refcount_t ref;
  21         struct kiocb *orig_iocb;
  22 };
  23
  24 static struct kmem_cache *ovl_aio_request_cachep;
  25
  26 static char ovl_whatisit(struct inode *inode, struct inode *realinode)
  27 {
  28         if (realinode != ovl_inode_upper(inode))
  29                 return 'l';
  30         if (ovl_has_upperdata(inode))
  31                 return 'u';
  32         else
  33                 return 'm';
  34 }
  35
  36 /* No atime modificaton nor notify on underlying */
  37 #define OVL_OPEN_FLAGS (O_NOATIME | FMODE_NONOTIFY)
  38
  39 static struct file *ovl_open_realfile(const struct file *file,
  40                                       struct inode *realinode)
  41 {
  42         struct inode *inode = file_inode(file);
  43         struct file *realfile;
  44         const struct cred *old_cred;
  45         int flags = file->f_flags | OVL_OPEN_FLAGS;
  46         int acc_mode = ACC_MODE(flags);
  47         int err;
  48
  49         if (flags & O_APPEND)
  50                 acc_mode |= MAY_APPEND;
  51
  52         old_cred = ovl_override_creds(inode->i_sb);
  53         err = inode_permission(realinode, MAY_OPEN | acc_mode);
  54         if (err) {
  55                 realfile = ERR_PTR(err);
  56         } else if (!inode_owner_or_capable(realinode)) {
  57                 realfile = ERR_PTR(-EPERM);
  58         } else {
  59                 realfile = open_with_fake_path(&file->f_path, flags, realinode,
  60                                                current_cred());
  61         }
  62         revert_creds(old_cred);
  63
  64         pr_debug("open(%p[%pD2/%c], 0%o) -> (%p, 0%o)\n",
  65                  file, file, ovl_whatisit(inode, realinode), file->f_flags,
  66                  realfile, IS_ERR(realfile) ? 0 : realfile->f_flags);
  67
  68         return realfile;
  69 }
  70
  71 #define OVL_SETFL_MASK (O_APPEND | O_NONBLOCK | O_NDELAY | O_DIRECT)
  72
  73 static int ovl_change_flags(struct file *file, unsigned int flags)
  74 {
  75         struct inode *inode = file_inode(file);
  76         int err;
  77
  78         flags |= OVL_OPEN_FLAGS;
  79
  80         /* If some flag changed that cannot be changed then something's amiss */
  81         if (WARN_ON((file->f_flags ^ flags) & ~OVL_SETFL_MASK))
  82                 return -EIO;
  83
  84         flags &= OVL_SETFL_MASK;
  85
  86         if (((flags ^ file->f_flags) & O_APPEND) && IS_APPEND(inode))
  87                 return -EPERM;
  88
  89         if (flags & O_DIRECT) {
  90                 if (!file->f_mapping->a_ops ||
  91                     !file->f_mapping->a_ops->direct_IO)
  92                         return -EINVAL;
  93         }
  94
  95         if (file->f_op->check_flags) {
  96                 err = file->f_op->check_flags(flags);
  97                 if (err)
  98                         return err;
  99         }
 100
 101         spin_lock(&file->f_lock);
 102         file->f_flags = (file->f_flags & ~OVL_SETFL_MASK) | flags;
 103         spin_unlock(&file->f_lock);
 104
 105         return 0;
 106 }
 107
 108 static int ovl_real_fdget_meta(const struct file *file, struct fd *real,
 109                                bool allow_meta)
 110 {
 111         struct inode *inode = file_inode(file);
 112         struct inode *realinode;
 113
 114         real->flags = 0;
 115         real->file = file->private_data;
 116
 117         if (allow_meta)
 118                 realinode = ovl_inode_real(inode);
 119         else
 120                 realinode = ovl_inode_realdata(inode);
 121
 122         /* Has it been copied up since we'd opened it? */
 123         if (unlikely(file_inode(real->file) != realinode)) {
 124                 real->flags = FDPUT_FPUT;
 125                 real->file = ovl_open_realfile(file, realinode);
 126
 127                 return PTR_ERR_OR_ZERO(real->file);
 128         }
 129
 130         /* Did the flags change since open? */
 131         if (unlikely((file->f_flags ^ real->file->f_flags) & ~OVL_OPEN_FLAGS))
 132                 return ovl_change_flags(real->file, file->f_flags);
 133
 134         return 0;
 135 }
 136
 137 static int ovl_real_fdget(const struct file *file, struct fd *real)
 138 {
 139         if (d_is_dir(file_dentry(file))) {
 140                 real->flags = 0;
 141                 real->file = ovl_dir_real_file(file, false);
 142
 143                 return PTR_ERR_OR_ZERO(real->file);
 144         }
 145
 146         return ovl_real_fdget_meta(file, real, false);
 147 }
 148
 149 static int ovl_open(struct inode *inode, struct file *file)
 150 {
 151         struct file *realfile;
 152         int err;
 153
 154         err = ovl_maybe_copy_up(file_dentry(file), file->f_flags);
 155         if (err)
 156                 return err;
 157
 158         /* No longer need these flags, so don't pass them on to underlying fs */
 159         file->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC);
 160
 161         realfile = ovl_open_realfile(file, ovl_inode_realdata(inode));
 162         if (IS_ERR(realfile))
 163                 return PTR_ERR(realfile);
 164
 165         file->private_data = realfile;
 166
 167         return 0;
 168 }
 169
 170 static int ovl_release(struct inode *inode, struct file *file)
 171 {
 172         fput(file->private_data);
 173
 174         return 0;
 175 }
 176
 177 static loff_t ovl_llseek(struct file *file, loff_t offset, int whence)
 178 {
 179         struct inode *inode = file_inode(file);
 180         struct fd real;
 181         const struct cred *old_cred;
 182         loff_t ret;
 183
 184         /*
 185          * The two special cases below do not need to involve real fs,
 186          * so we can optimizing concurrent callers.
 187          */
 188         if (offset == 0) {
 189                 if (whence == SEEK_CUR)
 190                         return file->f_pos;
 191
 192                 if (whence == SEEK_SET)
 193                         return vfs_setpos(file, 0, 0);
 194         }
 195
 196         ret = ovl_real_fdget(file, &real);
 197         if (ret)
 198                 return ret;
 199
 200         /*
 201          * Overlay file f_pos is the master copy that is preserved
 202          * through copy up and modified on read/write, but only real
 203          * fs knows how to SEEK_HOLE/SEEK_DATA and real fs may impose
 204          * limitations that are more strict than ->s_maxbytes for specific
 205          * files, so we use the real file to perform seeks.
 206          */
 207         ovl_inode_lock(inode);
 208         real.file->f_pos = file->f_pos;
 209
 210         old_cred = ovl_override_creds(inode->i_sb);
 211         ret = vfs_llseek(real.file, offset, whence);
 212         revert_creds(old_cred);
 213
 214         file->f_pos = real.file->f_pos;
 215         ovl_inode_unlock(inode);
 216
 217         fdput(real);
 218
 219         return ret;
 220 }
 221
 222 static void ovl_file_accessed(struct file *file)
 223 {
 224         struct inode *inode, *upperinode;
 225
 226         if (file->f_flags & O_NOATIME)
 227                 return;
 228
 229         inode = file_inode(file);
 230         upperinode = ovl_inode_upper(inode);
 231
 232         if (!upperinode)
 233                 return;
 234
 235         if ((!timespec64_equal(&inode->i_mtime, &upperinode->i_mtime) ||
 236              !timespec64_equal(&inode->i_ctime, &upperinode->i_ctime))) {
 237                 inode->i_mtime = upperinode->i_mtime;
 238                 inode->i_ctime = upperinode->i_ctime;
 239         }
 240
 241         touch_atime(&file->f_path);
 242 }
 243
 244 static rwf_t ovl_iocb_to_rwf(int ifl)
 245 {
 246         rwf_t flags = 0;
 247
 248         if (ifl & IOCB_NOWAIT)
 249                 flags |= RWF_NOWAIT;
 250         if (ifl & IOCB_HIPRI)
 251                 flags |= RWF_HIPRI;
 252         if (ifl & IOCB_DSYNC)
 253                 flags |= RWF_DSYNC;
 254         if (ifl & IOCB_SYNC)
 255                 flags |= RWF_SYNC;
 256
 257         return flags;
 258 }
 259
 260 static inline void ovl_aio_put(struct ovl_aio_req *aio_req)
 261 {
 262         if (refcount_dec_and_test(&aio_req->ref)) {
 263                 fput(aio_req->iocb.ki_filp);
 264                 kmem_cache_free(ovl_aio_request_cachep, aio_req);
 265         }
 266 }
 267
 268 static void ovl_aio_cleanup_handler(struct ovl_aio_req *aio_req)
 269 {
 270         struct kiocb *iocb = &aio_req->iocb;
 271         struct kiocb *orig_iocb = aio_req->orig_iocb;
 272
 273         if (iocb->ki_flags & IOCB_WRITE) {
 274                 struct inode *inode = file_inode(orig_iocb->ki_filp);
 275
 276                 /* Actually acquired in ovl_write_iter() */
 277                 __sb_writers_acquired(file_inode(iocb->ki_filp)->i_sb,
 278                                       SB_FREEZE_WRITE);
 279                 file_end_write(iocb->ki_filp);
 280                 ovl_copyattr(ovl_inode_real(inode), inode);
 281         }
 282
 283         orig_iocb->ki_pos = iocb->ki_pos;
 284         ovl_aio_put(aio_req);
 285 }
 286
 287 static void ovl_aio_rw_complete(struct kiocb *iocb, long res, long res2)
 288 {
 289         struct ovl_aio_req *aio_req = container_of(iocb,
 290                                                    struct ovl_aio_req, iocb);
 291         struct kiocb *orig_iocb = aio_req->orig_iocb;
 292
 293         ovl_aio_cleanup_handler(aio_req);
 294         orig_iocb->ki_complete(orig_iocb, res, res2);
 295 }
 296
 297 static ssize_t ovl_read_iter(struct kiocb *iocb, struct iov_iter *iter)
 298 {
 299         struct file *file = iocb->ki_filp;
 300         struct fd real;
 301         const struct cred *old_cred;
 302         ssize_t ret;
 303
 304         if (!iov_iter_count(iter))
 305                 return 0;
 306
 307         ret = ovl_real_fdget(file, &real);
 308         if (ret)
 309                 return ret;
 310
 311         ret = -EINVAL;
 312         if (iocb->ki_flags & IOCB_DIRECT &&
 313             (!real.file->f_mapping->a_ops ||
 314              !real.file->f_mapping->a_ops->direct_IO))
 315                 goto out_fdput;
 316
 317         old_cred = ovl_override_creds(file_inode(file)->i_sb);
 318         if (is_sync_kiocb(iocb)) {
 319                 ret = vfs_iter_read(real.file, iter, &iocb->ki_pos,
 320                                     ovl_iocb_to_rwf(iocb->ki_flags));
 321         } else {
 322                 struct ovl_aio_req *aio_req;
 323
 324                 ret = -ENOMEM;
 325                 aio_req = kmem_cache_zalloc(ovl_aio_request_cachep, GFP_KERNEL);
 326                 if (!aio_req)
 327                         goto out;
 328
 329                 real.flags = 0;
 330                 aio_req->orig_iocb = iocb;
 331                 kiocb_clone(&aio_req->iocb, iocb, get_file(real.file));
 332                 aio_req->iocb.ki_complete = ovl_aio_rw_complete;
 333                 refcount_set(&aio_req->ref, 2);
 334                 ret = vfs_iocb_iter_read(real.file, &aio_req->iocb, iter);
 335                 ovl_aio_put(aio_req);
 336                 if (ret != -EIOCBQUEUED)
 337                         ovl_aio_cleanup_handler(aio_req);
 338         }
 339 out:
 340         revert_creds(old_cred);
 341         ovl_file_accessed(file);
 342 out_fdput:
 343         fdput(real);
 344
 345         return ret;
 346 }
 347
 348 static ssize_t ovl_write_iter(struct kiocb *iocb, struct iov_iter *iter)
 349 {
 350         struct file *file = iocb->ki_filp;
 351         struct inode *inode = file_inode(file);
 352         struct fd real;
 353         const struct cred *old_cred;
 354         ssize_t ret;
 355         int ifl = iocb->ki_flags;
 356
 357         if (!iov_iter_count(iter))
 358                 return 0;
 359
 360         inode_lock(inode);
 361         /* Update mode */
 362         ovl_copyattr(ovl_inode_real(inode), inode);
 363         ret = file_remove_privs(file);
 364         if (ret)
 365                 goto out_unlock;
 366
 367         ret = ovl_real_fdget(file, &real);
 368         if (ret)
 369                 goto out_unlock;
 370
 371         ret = -EINVAL;
 372         if (iocb->ki_flags & IOCB_DIRECT &&
 373             (!real.file->f_mapping->a_ops ||
 374              !real.file->f_mapping->a_ops->direct_IO))
 375                 goto out_fdput;
 376
 377         if (!ovl_should_sync(OVL_FS(inode->i_sb)))
 378                 ifl &= ~(IOCB_DSYNC | IOCB_SYNC);
 379
 380         old_cred = ovl_override_creds(file_inode(file)->i_sb);
 381         if (is_sync_kiocb(iocb)) {
 382                 file_start_write(real.file);
 383                 ret = vfs_iter_write(real.file, iter, &iocb->ki_pos,
 384                                      ovl_iocb_to_rwf(ifl));
 385                 file_end_write(real.file);
 386                 /* Update size */
 387                 ovl_copyattr(ovl_inode_real(inode), inode);
 388         } else {
 389                 struct ovl_aio_req *aio_req;
 390
 391                 ret = -ENOMEM;
 392                 aio_req = kmem_cache_zalloc(ovl_aio_request_cachep, GFP_KERNEL);
 393                 if (!aio_req)
 394                         goto out;
 395
 396                 file_start_write(real.file);
 397                 /* Pacify lockdep, same trick as done in aio_write() */
 398                 __sb_writers_release(file_inode(real.file)->i_sb,
 399                                      SB_FREEZE_WRITE);
 400                 real.flags = 0;
 401                 aio_req->orig_iocb = iocb;
 402                 kiocb_clone(&aio_req->iocb, iocb, get_file(real.file));
 403                 aio_req->iocb.ki_flags = ifl;
 404                 aio_req->iocb.ki_complete = ovl_aio_rw_complete;
 405                 refcount_set(&aio_req->ref, 2);
 406                 ret = vfs_iocb_iter_write(real.file, &aio_req->iocb, iter);
 407                 ovl_aio_put(aio_req);
 408                 if (ret != -EIOCBQUEUED)
 409                         ovl_aio_cleanup_handler(aio_req);
 410         }
 411 out:
 412         revert_creds(old_cred);
 413 out_fdput:
 414         fdput(real);
 415
 416 out_unlock:
 417         inode_unlock(inode);
 418
 419         return ret;
 420 }
 421
 422 /*
 423  * Calling iter_file_splice_write() directly from overlay's f_op may deadlock
 424  * due to lock order inversion between pipe->mutex in iter_file_splice_write()
 425  * and file_start_write(real.file) in ovl_write_iter().
 426  *
 427  * So do everything ovl_write_iter() does and call iter_file_splice_write() on
 428  * the real file.
 429  */
 430 static ssize_t ovl_splice_write(struct pipe_inode_info *pipe, struct file *out,
 431                                 loff_t *ppos, size_t len, unsigned int flags)
 432 {
 433         struct fd real;
 434         const struct cred *old_cred;
 435         struct inode *inode = file_inode(out);
 436         struct inode *realinode = ovl_inode_real(inode);
 437         ssize_t ret;
 438
 439         inode_lock(inode);
 440         /* Update mode */
 441         ovl_copyattr(realinode, inode);
 442         ret = file_remove_privs(out);
 443         if (ret)
 444                 goto out_unlock;
 445
 446         ret = ovl_real_fdget(out, &real);
 447         if (ret)
 448                 goto out_unlock;
 449
 450         old_cred = ovl_override_creds(inode->i_sb);
 451         file_start_write(real.file);
 452
 453         ret = iter_file_splice_write(pipe, real.file, ppos, len, flags);
 454
 455         file_end_write(real.file);
 456         /* Update size */
 457         ovl_copyattr(realinode, inode);
 458         revert_creds(old_cred);
 459         fdput(real);
 460
 461 out_unlock:
 462         inode_unlock(inode);
 463
 464         return ret;
 465 }
 466
 467 static int ovl_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 468 {
 469         struct fd real;
 470         const struct cred *old_cred;
 471         int ret;
 472
 473         ret = ovl_sync_status(OVL_FS(file_inode(file)->i_sb));
 474         if (ret <= 0)
 475                 return ret;
 476
 477         ret = ovl_real_fdget_meta(file, &real, !datasync);
 478         if (ret)
 479                 return ret;
 480
 481         /* Don't sync lower file for fear of receiving EROFS error */
 482         if (file_inode(real.file) == ovl_inode_upper(file_inode(file))) {
 483                 old_cred = ovl_override_creds(file_inode(file)->i_sb);
 484                 ret = vfs_fsync_range(real.file, start, end, datasync);
 485                 revert_creds(old_cred);
 486         }
 487
 488         fdput(real);
 489
 490         return ret;
 491 }
 492
 493 static int ovl_mmap(struct file *file, struct vm_area_struct *vma)
 494 {
 495         struct file *realfile = file->private_data;
 496         const struct cred *old_cred;
 497         int ret;
 498
 499         if (!realfile->f_op->mmap)
 500                 return -ENODEV;
 501
 502         if (WARN_ON(file != vma->vm_file))
 503                 return -EIO;
 504
 505         vma->vm_file = get_file(realfile);
 506
 507         old_cred = ovl_override_creds(file_inode(file)->i_sb);
 508         ret = call_mmap(vma->vm_file, vma);
 509         revert_creds(old_cred);
 510
 511         if (ret) {
 512                 /* Drop reference count from new vm_file value */
 513                 fput(realfile);
 514         } else {
 515                 /* Drop reference count from previous vm_file value */
 516                 fput(file);
 517         }
 518
 519         ovl_file_accessed(file);
 520
 521         return ret;
 522 }
 523
 524 static long ovl_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
 525 {
 526         struct inode *inode = file_inode(file);
 527         struct fd real;
 528         const struct cred *old_cred;
 529         int ret;
 530
 531         inode_lock(inode);
 532         /* Update mode */
 533         ovl_copyattr(ovl_inode_real(inode), inode);
 534         ret = file_remove_privs(file);
 535         if (ret)
 536                 goto out_unlock;
 537
 538         ret = ovl_real_fdget(file, &real);
 539         if (ret)
 540                 goto out_unlock;
 541
 542         old_cred = ovl_override_creds(file_inode(file)->i_sb);
 543         ret = vfs_fallocate(real.file, mode, offset, len);
 544         revert_creds(old_cred);
 545
 546         /* Update size */
 547         ovl_copyattr(ovl_inode_real(inode), inode);
 548
 549         fdput(real);
 550
 551 out_unlock:
 552         inode_unlock(inode);
 553
 554         return ret;
 555 }
 556
 557 static int ovl_fadvise(struct file *file, loff_t offset, loff_t len, int advice)
 558 {
 559         struct fd real;
 560         const struct cred *old_cred;
 561         int ret;
 562
 563         ret = ovl_real_fdget(file, &real);
 564         if (ret)
 565                 return ret;
 566
 567         old_cred = ovl_override_creds(file_inode(file)->i_sb);
 568         ret = vfs_fadvise(real.file, offset, len, advice);
 569         revert_creds(old_cred);
 570
 571         fdput(real);
 572
 573         return ret;
 574 }
 575
 576 static long ovl_real_ioctl(struct file *file, unsigned int cmd,
 577                            unsigned long arg)
 578 {
 579         struct fd real;
 580         long ret;
 581
 582         ret = ovl_real_fdget(file, &real);
 583         if (ret)
 584                 return ret;
 585
 586         ret = security_file_ioctl(real.file, cmd, arg);
 587         if (!ret) {
 588                 /*
 589                  * Don't override creds, since we currently can't safely check
 590                  * permissions before doing so.
 591                  */
 592                 ret = vfs_ioctl(real.file, cmd, arg);
 593         }
 594
 595         fdput(real);
 596
 597         return ret;
 598 }
 599
 600 static long ovl_ioctl_set_flags(struct file *file, unsigned int cmd,
 601                                 unsigned long arg)
 602 {
 603         long ret;
 604         struct inode *inode = file_inode(file);
 605
 606         if (!inode_owner_or_capable(inode))
 607                 return -EACCES;
 608
 609         ret = mnt_want_write_file(file);
 610         if (ret)
 611                 return ret;
 612
 613         inode_lock(inode);
 614
 615         /*
 616          * Prevent copy up if immutable and has no CAP_LINUX_IMMUTABLE
 617          * capability.
 618          */
 619         ret = -EPERM;
 620         if (!ovl_has_upperdata(inode) && IS_IMMUTABLE(inode) &&
 621             !capable(CAP_LINUX_IMMUTABLE))
 622                 goto unlock;
 623
 624         ret = ovl_maybe_copy_up(file_dentry(file), O_WRONLY);
 625         if (ret)
 626                 goto unlock;
 627
 628         ret = ovl_real_ioctl(file, cmd, arg);
 629
 630         ovl_copyflags(ovl_inode_real(inode), inode);
 631 unlock:
 632         inode_unlock(inode);
 633
 634         mnt_drop_write_file(file);
 635
 636         return ret;
 637
 638 }
 639
 640 long ovl_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 641 {
 642         long ret;
 643
 644         switch (cmd) {
 645         case FS_IOC_GETFLAGS:
 646         case FS_IOC_FSGETXATTR:
 647                 ret = ovl_real_ioctl(file, cmd, arg);
 648                 break;
 649
 650         case FS_IOC_FSSETXATTR:
 651         case FS_IOC_SETFLAGS:
 652                 ret = ovl_ioctl_set_flags(file, cmd, arg);
 653                 break;
 654
 655         default:
 656                 ret = -ENOTTY;
 657         }
 658
 659         return ret;
 660 }
 661
 662 #ifdef CONFIG_COMPAT
 663 long ovl_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 664 {
 665         switch (cmd) {
 666         case FS_IOC32_GETFLAGS:
 667                 cmd = FS_IOC_GETFLAGS;
 668                 break;
 669
 670         case FS_IOC32_SETFLAGS:
 671                 cmd = FS_IOC_SETFLAGS;
 672                 break;
 673
 674         default:
 675                 return -ENOIOCTLCMD;
 676         }
 677
 678         return ovl_ioctl(file, cmd, arg);
 679 }
 680 #endif
 681
 682 enum ovl_copyop {
 683         OVL_COPY,
 684         OVL_CLONE,
 685         OVL_DEDUPE,
 686 };
 687
 688 static loff_t ovl_copyfile(struct file *file_in, loff_t pos_in,
 689                             struct file *file_out, loff_t pos_out,
 690                             loff_t len, unsigned int flags, enum ovl_copyop op)
 691 {
 692         struct inode *inode_out = file_inode(file_out);
 693         struct fd real_in, real_out;
 694         const struct cred *old_cred;
 695         loff_t ret;
 696
 697         inode_lock(inode_out);
 698         if (op != OVL_DEDUPE) {
 699                 /* Update mode */
 700                 ovl_copyattr(ovl_inode_real(inode_out), inode_out);
 701                 ret = file_remove_privs(file_out);
 702                 if (ret)
 703                         goto out_unlock;
 704         }
 705
 706         ret = ovl_real_fdget(file_out, &real_out);
 707         if (ret)
 708                 goto out_unlock;
 709
 710         ret = ovl_real_fdget(file_in, &real_in);
 711         if (ret) {
 712                 fdput(real_out);
 713                 goto out_unlock;
 714         }
 715
 716         old_cred = ovl_override_creds(file_inode(file_out)->i_sb);
 717         switch (op) {
 718         case OVL_COPY:
 719                 ret = vfs_copy_file_range(real_in.file, pos_in,
 720                                           real_out.file, pos_out, len, flags);
 721                 break;
 722
 723         case OVL_CLONE:
 724                 ret = vfs_clone_file_range(real_in.file, pos_in,
 725                                            real_out.file, pos_out, len, flags);
 726                 break;
 727
 728         case OVL_DEDUPE:
 729                 ret = vfs_dedupe_file_range_one(real_in.file, pos_in,
 730                                                 real_out.file, pos_out, len,
 731                                                 flags);
 732                 break;
 733         }
 734         revert_creds(old_cred);
 735
 736         /* Update size */
 737         ovl_copyattr(ovl_inode_real(inode_out), inode_out);
 738
 739         fdput(real_in);
 740         fdput(real_out);
 741
 742 out_unlock:
 743         inode_unlock(inode_out);
 744
 745         return ret;
 746 }
 747
 748 static ssize_t ovl_copy_file_range(struct file *file_in, loff_t pos_in,
 749                                    struct file *file_out, loff_t pos_out,
 750                                    size_t len, unsigned int flags)
 751 {
 752         return ovl_copyfile(file_in, pos_in, file_out, pos_out, len, flags,
 753                             OVL_COPY);
 754 }
 755
 756 static loff_t ovl_remap_file_range(struct file *file_in, loff_t pos_in,
 757                                    struct file *file_out, loff_t pos_out,
 758                                    loff_t len, unsigned int remap_flags)
 759 {
 760         enum ovl_copyop op;
 761
 762         if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY))
 763                 return -EINVAL;
 764
 765         if (remap_flags & REMAP_FILE_DEDUP)
 766                 op = OVL_DEDUPE;
 767         else
 768                 op = OVL_CLONE;
 769
 770         /*
 771          * Don't copy up because of a dedupe request, this wouldn't make sense
 772          * most of the time (data would be duplicated instead of deduplicated).
 773          */
 774         if (op == OVL_DEDUPE &&
 775             (!ovl_inode_upper(file_inode(file_in)) ||
 776              !ovl_inode_upper(file_inode(file_out))))
 777                 return -EPERM;
 778
 779         return ovl_copyfile(file_in, pos_in, file_out, pos_out, len,
 780                             remap_flags, op);
 781 }
 782
 783 const struct file_operations ovl_file_operations = {
 784         .open           = ovl_open,
 785         .release        = ovl_release,
 786         .llseek         = ovl_llseek,
 787         .read_iter      = ovl_read_iter,
 788         .write_iter     = ovl_write_iter,
 789         .fsync          = ovl_fsync,
 790         .mmap           = ovl_mmap,
 791         .fallocate      = ovl_fallocate,
 792         .fadvise        = ovl_fadvise,
 793         .unlocked_ioctl = ovl_ioctl,
 794 #ifdef CONFIG_COMPAT
 795         .compat_ioctl   = ovl_compat_ioctl,
 796 #endif
 797         .splice_read    = generic_file_splice_read,
 798         .splice_write   = ovl_splice_write,
 799
 800         .copy_file_range        = ovl_copy_file_range,
 801         .remap_file_range       = ovl_remap_file_range,
 802 };
 803
 804 int __init ovl_aio_request_cache_init(void)
 805 {
 806         ovl_aio_request_cachep = kmem_cache_create("ovl_aio_req",
 807                                                    sizeof(struct ovl_aio_req),
 808                                                    0, SLAB_HWCACHE_ALIGN, NULL);
 809         if (!ovl_aio_request_cachep)
 810                 return -ENOMEM;
 811
 812         return 0;
 813 }
 814
 815 void ovl_aio_request_cache_destroy(void)
 816 {
 817         kmem_cache_destroy(ovl_aio_request_cachep);
 818 }