mm/memfd.c

   1 /*
   2  * memfd_create system call and file sealing support
   3  *
   4  * Code was originally included in shmem.c, and broken out to facilitate
   5  * use by hugetlbfs as well as tmpfs.
   6  *
   7  * This file is released under the GPL.
   8  */
   9
  10 #include <linux/fs.h>
  11 #include <linux/vfs.h>
  12 #include <linux/pagemap.h>
  13 #include <linux/file.h>
  14 #include <linux/mm.h>
  15 #include <linux/sched/signal.h>
  16 #include <linux/khugepaged.h>
  17 #include <linux/syscalls.h>
  18 #include <linux/hugetlb.h>
  19 #include <linux/shmem_fs.h>
  20 #include <linux/memfd.h>
  21 #include <uapi/linux/memfd.h>
  22
  23 /*
  24  * We need a tag: a new tag would expand every radix_tree_node by 8 bytes,
  25  * so reuse a tag which we firmly believe is never set or cleared on tmpfs
  26  * or hugetlbfs because they are memory only filesystems.
  27  */
  28 #define MEMFD_TAG_PINNED        PAGECACHE_TAG_TOWRITE
  29 #define LAST_SCAN               4       /* about 150ms max */
  30
  31 static void memfd_tag_pins(struct address_space *mapping)
  32 {
  33         struct radix_tree_iter iter;
  34         void __rcu **slot;
  35         pgoff_t start;
  36         struct page *page;
  37         int latency = 0;
  38         int cache_count;
  39
  40         lru_add_drain();
  41         start = 0;
  42
  43         xa_lock_irq(&mapping->i_pages);
  44         radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
  45                 cache_count = 1;
  46                 page = radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock);
  47                 if (!page || radix_tree_exception(page) || PageTail(page)) {
  48                         if (radix_tree_deref_retry(page)) {
  49                                 slot = radix_tree_iter_retry(&iter);
  50                                 continue;
  51                         }
  52                 } else {
  53                         if (PageTransHuge(page) && !PageHuge(page))
  54                                 cache_count = HPAGE_PMD_NR;
  55                         if (cache_count !=
  56                             page_count(page) - total_mapcount(page)) {
  57                                 radix_tree_tag_set(&mapping->i_pages,
  58                                                 iter.index, MEMFD_TAG_PINNED);
  59                         }
  60                 }
  61
  62                 latency += cache_count;
  63                 if (latency < 1024)
  64                         continue;
  65                 latency = 0;
  66
  67                 slot = radix_tree_iter_resume(slot, &iter);
  68                 xa_unlock_irq(&mapping->i_pages);
  69                 cond_resched();
  70                 xa_lock_irq(&mapping->i_pages);
  71         }
  72         xa_unlock_irq(&mapping->i_pages);
  73 }
  74
  75 /*
  76  * Setting SEAL_WRITE requires us to verify there's no pending writer. However,
  77  * via get_user_pages(), drivers might have some pending I/O without any active
  78  * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages
  79  * and see whether it has an elevated ref-count. If so, we tag them and wait for
  80  * them to be dropped.
  81  * The caller must guarantee that no new user will acquire writable references
  82  * to those pages to avoid races.
  83  */
  84 static int memfd_wait_for_pins(struct address_space *mapping)
  85 {
  86         struct radix_tree_iter iter;
  87         void __rcu **slot;
  88         pgoff_t start;
  89         struct page *page;
  90         int error, scan;
  91         int cache_count;
  92
  93         memfd_tag_pins(mapping);
  94
  95         error = 0;
  96         for (scan = 0; scan <= LAST_SCAN; scan++) {
  97                 if (!radix_tree_tagged(&mapping->i_pages, MEMFD_TAG_PINNED))
  98                         break;
  99
 100                 if (!scan)
 101                         lru_add_drain_all();
 102                 else if (schedule_timeout_killable((HZ << scan) / 200))
 103                         scan = LAST_SCAN;
 104
 105                 start = 0;
 106                 rcu_read_lock();
 107                 radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter,
 108                                            start, MEMFD_TAG_PINNED) {
 109
 110                         page = radix_tree_deref_slot(slot);
 111                         if (radix_tree_exception(page)) {
 112                                 if (radix_tree_deref_retry(page)) {
 113                                         slot = radix_tree_iter_retry(&iter);
 114                                         continue;
 115                                 }
 116
 117                                 page = NULL;
 118                         }
 119
 120                         cache_count = 1;
 121                         if (page && PageTransHuge(page) && !PageHuge(page))
 122                                 cache_count = HPAGE_PMD_NR;
 123
 124                         if (page && cache_count !=
 125                             page_count(page) - total_mapcount(page)) {
 126                                 if (scan < LAST_SCAN)
 127                                         goto continue_resched;
 128
 129                                 /*
 130                                  * On the last scan, we clean up all those tags
 131                                  * we inserted; but make a note that we still
 132                                  * found pages pinned.
 133                                  */
 134                                 error = -EBUSY;
 135                         }
 136
 137                         xa_lock_irq(&mapping->i_pages);
 138                         radix_tree_tag_clear(&mapping->i_pages,
 139                                              iter.index, MEMFD_TAG_PINNED);
 140                         xa_unlock_irq(&mapping->i_pages);
 141 continue_resched:
 142                         if (need_resched()) {
 143                                 slot = radix_tree_iter_resume(slot, &iter);
 144                                 cond_resched_rcu();
 145                         }
 146                 }
 147                 rcu_read_unlock();
 148         }
 149
 150         return error;
 151 }
 152
 153 static unsigned int *memfd_file_seals_ptr(struct file *file)
 154 {
 155         if (shmem_file(file))
 156                 return &SHMEM_I(file_inode(file))->seals;
 157
 158 #ifdef CONFIG_HUGETLBFS
 159         if (is_file_hugepages(file))
 160                 return &HUGETLBFS_I(file_inode(file))->seals;
 161 #endif
 162
 163         return NULL;
 164 }
 165
 166 #define F_ALL_SEALS (F_SEAL_SEAL | \
 167                      F_SEAL_SHRINK | \
 168                      F_SEAL_GROW | \
 169                      F_SEAL_WRITE)
 170
 171 static int memfd_add_seals(struct file *file, unsigned int seals)
 172 {
 173         struct inode *inode = file_inode(file);
 174         unsigned int *file_seals;
 175         int error;
 176
 177         /*
 178          * SEALING
 179          * Sealing allows multiple parties to share a tmpfs or hugetlbfs file
 180          * but restrict access to a specific subset of file operations. Seals
 181          * can only be added, but never removed. This way, mutually untrusted
 182          * parties can share common memory regions with a well-defined policy.
 183          * A malicious peer can thus never perform unwanted operations on a
 184          * shared object.
 185          *
 186          * Seals are only supported on special tmpfs or hugetlbfs files and
 187          * always affect the whole underlying inode. Once a seal is set, it
 188          * may prevent some kinds of access to the file. Currently, the
 189          * following seals are defined:
 190          *   SEAL_SEAL: Prevent further seals from being set on this file
 191          *   SEAL_SHRINK: Prevent the file from shrinking
 192          *   SEAL_GROW: Prevent the file from growing
 193          *   SEAL_WRITE: Prevent write access to the file
 194          *
 195          * As we don't require any trust relationship between two parties, we
 196          * must prevent seals from being removed. Therefore, sealing a file
 197          * only adds a given set of seals to the file, it never touches
 198          * existing seals. Furthermore, the "setting seals"-operation can be
 199          * sealed itself, which basically prevents any further seal from being
 200          * added.
 201          *
 202          * Semantics of sealing are only defined on volatile files. Only
 203          * anonymous tmpfs and hugetlbfs files support sealing. More
 204          * importantly, seals are never written to disk. Therefore, there's
 205          * no plan to support it on other file types.
 206          */
 207
 208         if (!(file->f_mode & FMODE_WRITE))
 209                 return -EPERM;
 210         if (seals & ~(unsigned int)F_ALL_SEALS)
 211                 return -EINVAL;
 212
 213         inode_lock(inode);
 214
 215         file_seals = memfd_file_seals_ptr(file);
 216         if (!file_seals) {
 217                 error = -EINVAL;
 218                 goto unlock;
 219         }
 220
 221         if (*file_seals & F_SEAL_SEAL) {
 222                 error = -EPERM;
 223                 goto unlock;
 224         }
 225
 226         if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) {
 227                 error = mapping_deny_writable(file->f_mapping);
 228                 if (error)
 229                         goto unlock;
 230
 231                 error = memfd_wait_for_pins(file->f_mapping);
 232                 if (error) {
 233                         mapping_allow_writable(file->f_mapping);
 234                         goto unlock;
 235                 }
 236         }
 237
 238         *file_seals |= seals;
 239         error = 0;
 240
 241 unlock:
 242         inode_unlock(inode);
 243         return error;
 244 }
 245
 246 static int memfd_get_seals(struct file *file)
 247 {
 248         unsigned int *seals = memfd_file_seals_ptr(file);
 249
 250         return seals ? *seals : -EINVAL;
 251 }
 252
 253 long memfd_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
 254 {
 255         long error;
 256
 257         switch (cmd) {
 258         case F_ADD_SEALS:
 259                 /* disallow upper 32bit */
 260                 if (arg > UINT_MAX)
 261                         return -EINVAL;
 262
 263                 error = memfd_add_seals(file, arg);
 264                 break;
 265         case F_GET_SEALS:
 266                 error = memfd_get_seals(file);
 267                 break;
 268         default:
 269                 error = -EINVAL;
 270                 break;
 271         }
 272
 273         return error;
 274 }
 275
 276 #define MFD_NAME_PREFIX "memfd:"
 277 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1)
 278 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN)
 279
 280 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB)
 281
 282 SYSCALL_DEFINE2(memfd_create,
 283                 const char __user *, uname,
 284                 unsigned int, flags)
 285 {
 286         unsigned int *file_seals;
 287         struct file *file;
 288         int fd, error;
 289         char *name;
 290         long len;
 291
 292         if (!(flags & MFD_HUGETLB)) {
 293                 if (flags & ~(unsigned int)MFD_ALL_FLAGS)
 294                         return -EINVAL;
 295         } else {
 296                 /* Allow huge page size encoding in flags. */
 297                 if (flags & ~(unsigned int)(MFD_ALL_FLAGS |
 298                                 (MFD_HUGE_MASK << MFD_HUGE_SHIFT)))
 299                         return -EINVAL;
 300         }
 301
 302         /* length includes terminating zero */
 303         len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1);
 304         if (len <= 0)
 305                 return -EFAULT;
 306         if (len > MFD_NAME_MAX_LEN + 1)
 307                 return -EINVAL;
 308
 309         name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_KERNEL);
 310         if (!name)
 311                 return -ENOMEM;
 312
 313         strcpy(name, MFD_NAME_PREFIX);
 314         if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) {
 315                 error = -EFAULT;
 316                 goto err_name;
 317         }
 318
 319         /* terminating-zero may have changed after strnlen_user() returned */
 320         if (name[len + MFD_NAME_PREFIX_LEN - 1]) {
 321                 error = -EFAULT;
 322                 goto err_name;
 323         }
 324
 325         fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0);
 326         if (fd < 0) {
 327                 error = fd;
 328                 goto err_name;
 329         }
 330
 331         if (flags & MFD_HUGETLB) {
 332                 struct user_struct *user = NULL;
 333
 334                 file = hugetlb_file_setup(name, 0, VM_NORESERVE, &user,
 335                                         HUGETLB_ANONHUGE_INODE,
 336                                         (flags >> MFD_HUGE_SHIFT) &
 337                                         MFD_HUGE_MASK);
 338         } else
 339                 file = shmem_file_setup(name, 0, VM_NORESERVE);
 340         if (IS_ERR(file)) {
 341                 error = PTR_ERR(file);
 342                 goto err_fd;
 343         }
 344         file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
 345         file->f_flags |= O_LARGEFILE;
 346
 347         if (flags & MFD_ALLOW_SEALING) {
 348                 file_seals = memfd_file_seals_ptr(file);
 349                 *file_seals &= ~F_SEAL_SEAL;
 350         }
 351
 352         fd_install(fd, file);
 353         kfree(name);
 354         return fd;
 355
 356 err_fd:
 357         put_unused_fd(fd);
 358 err_name:
 359         kfree(name);
 360         return error;
 361 }