ipc/shm.c

   1 /*
   2  * linux/ipc/shm.c
   3  * Copyright (C) 1992, 1993 Krishna Balasubramanian
   4  *       Many improvements/fixes by Bruno Haible.
   5  * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
   6  * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
   7  *
   8  * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
   9  * BIGMEM support, Andrea Arcangeli <andrea@suse.de>
  10  * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr>
  11  * HIGHMEM support, Ingo Molnar <mingo@redhat.com>
  12  * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com>
  13  * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com>
  14  * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com>
  15  *
  16  * support for audit of ipc object properties and permission changes
  17  * Dustin Kirkland <dustin.kirkland@us.ibm.com>
  18  *
  19  * namespaces support
  20  * OpenVZ, SWsoft Inc.
  21  * Pavel Emelianov <xemul@openvz.org>
  22  *
  23  * Better ipc lock (kern_ipc_perm.lock) handling
  24  * Davidlohr Bueso <davidlohr.bueso@hp.com>, June 2013.
  25  */
  26
  27 #include <linux/slab.h>
  28 #include <linux/mm.h>
  29 #include <linux/hugetlb.h>
  30 #include <linux/shm.h>
  31 #include <linux/init.h>
  32 #include <linux/file.h>
  33 #include <linux/mman.h>
  34 #include <linux/shmem_fs.h>
  35 #include <linux/security.h>
  36 #include <linux/syscalls.h>
  37 #include <linux/audit.h>
  38 #include <linux/capability.h>
  39 #include <linux/ptrace.h>
  40 #include <linux/seq_file.h>
  41 #include <linux/rwsem.h>
  42 #include <linux/nsproxy.h>
  43 #include <linux/mount.h>
  44 #include <linux/ipc_namespace.h>
  45
  46 #include <linux/uaccess.h>
  47
  48 #include "util.h"
  49
  50 struct shm_file_data {
  51         int id;
  52         struct ipc_namespace *ns;
  53         struct file *file;
  54         const struct vm_operations_struct *vm_ops;
  55 };
  56
  57 #define shm_file_data(file) (*((struct shm_file_data **)&(file)->private_data))
  58
  59 static const struct file_operations shm_file_operations;
  60 static const struct vm_operations_struct shm_vm_ops;
  61
  62 #define shm_ids(ns)     ((ns)->ids[IPC_SHM_IDS])
  63
  64 #define shm_unlock(shp)                 \
  65         ipc_unlock(&(shp)->shm_perm)
  66
  67 static int newseg(struct ipc_namespace *, struct ipc_params *);
  68 static void shm_open(struct vm_area_struct *vma);
  69 static void shm_close(struct vm_area_struct *vma);
  70 static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);
  71 #ifdef CONFIG_PROC_FS
  72 static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
  73 #endif
  74
  75 void shm_init_ns(struct ipc_namespace *ns)
  76 {
  77         ns->shm_ctlmax = SHMMAX;
  78         ns->shm_ctlall = SHMALL;
  79         ns->shm_ctlmni = SHMMNI;
  80         ns->shm_rmid_forced = 0;
  81         ns->shm_tot = 0;
  82         ipc_init_ids(&shm_ids(ns));
  83 }
  84
  85 /*
  86  * Called with shm_ids.rwsem (writer) and the shp structure locked.
  87  * Only shm_ids.rwsem remains locked on exit.
  88  */
  89 static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
  90 {
  91         struct shmid_kernel *shp;
  92         shp = container_of(ipcp, struct shmid_kernel, shm_perm);
  93         WARN_ON(ns != shp->ns);
  94
  95         if (shp->shm_nattch) {
  96                 shp->shm_perm.mode |= SHM_DEST;
  97                 /* Do not find it any more */
  98                 shp->shm_perm.key = IPC_PRIVATE;
  99                 shm_unlock(shp);
 100         } else
 101                 shm_destroy(ns, shp);
 102 }
 103
 104 #ifdef CONFIG_IPC_NS
 105 void shm_exit_ns(struct ipc_namespace *ns)
 106 {
 107         free_ipcs(ns, &shm_ids(ns), do_shm_rmid);
 108         idr_destroy(&ns->ids[IPC_SHM_IDS].ipcs_idr);
 109 }
 110 #endif
 111
 112 static int __init ipc_ns_init(void)
 113 {
 114         shm_init_ns(&init_ipc_ns);
 115         return 0;
 116 }
 117
 118 pure_initcall(ipc_ns_init);
 119
 120 void __init shm_init(void)
 121 {
 122         ipc_init_proc_interface("sysvipc/shm",
 123 #if BITS_PER_LONG <= 32
 124                                 "       key      shmid perms       size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime        rss       swap\n",
 125 #else
 126                                 "       key      shmid perms                  size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime                   rss                  swap\n",
 127 #endif
 128                                 IPC_SHM_IDS, sysvipc_shm_proc_show);
 129 }
 130
 131 static inline struct shmid_kernel *shm_obtain_object(struct ipc_namespace *ns, int id)
 132 {
 133         struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
 134
 135         if (IS_ERR(ipcp))
 136                 return ERR_CAST(ipcp);
 137
 138         return container_of(ipcp, struct shmid_kernel, shm_perm);
 139 }
 140
 141 static inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace *ns, int id)
 142 {
 143         struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&shm_ids(ns), id);
 144
 145         if (IS_ERR(ipcp))
 146                 return ERR_CAST(ipcp);
 147
 148         return container_of(ipcp, struct shmid_kernel, shm_perm);
 149 }
 150
 151 /*
 152  * shm_lock_(check_) routines are called in the paths where the rwsem
 153  * is not necessarily held.
 154  */
 155 static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
 156 {
 157         struct kern_ipc_perm *ipcp = ipc_lock(&shm_ids(ns), id);
 158
 159         /*
 160          * Callers of shm_lock() must validate the status of the returned ipc
 161          * object pointer (as returned by ipc_lock()), and error out as
 162          * appropriate.
 163          */
 164         if (IS_ERR(ipcp))
 165                 return (void *)ipcp;
 166         return container_of(ipcp, struct shmid_kernel, shm_perm);
 167 }
 168
 169 static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp)
 170 {
 171         rcu_read_lock();
 172         ipc_lock_object(&ipcp->shm_perm);
 173 }
 174
 175 static void shm_rcu_free(struct rcu_head *head)
 176 {
 177         struct ipc_rcu *p = container_of(head, struct ipc_rcu, rcu);
 178         struct shmid_kernel *shp = ipc_rcu_to_struct(p);
 179
 180         security_shm_free(shp);
 181         ipc_rcu_free(head);
 182 }
 183
 184 /*
 185  * It has to be called with shp locked.
 186  * It must be called before ipc_rmid()
 187  */
 188 static inline void shm_clist_rm(struct shmid_kernel *shp)
 189 {
 190         struct task_struct *creator;
 191
 192         /* ensure that shm_creator does not disappear */
 193         rcu_read_lock();
 194
 195         /*
 196          * A concurrent exit_shm may do a list_del_init() as well.
 197          * Just do nothing if exit_shm already did the work
 198          */
 199         if (!list_empty(&shp->shm_clist)) {
 200                 /*
 201                  * shp->shm_creator is guaranteed to be valid *only*
 202                  * if shp->shm_clist is not empty.
 203                  */
 204                 creator = shp->shm_creator;
 205
 206                 task_lock(creator);
 207                 /*
 208                  * list_del_init() is a nop if the entry was already removed
 209                  * from the list.
 210                  */
 211                 list_del_init(&shp->shm_clist);
 212                 task_unlock(creator);
 213         }
 214         rcu_read_unlock();
 215 }
 216
 217 static inline void shm_rmid(struct shmid_kernel *s)
 218 {
 219         shm_clist_rm(s);
 220         ipc_rmid(&shm_ids(s->ns), &s->shm_perm);
 221 }
 222
 223
 224 static int __shm_open(struct vm_area_struct *vma)
 225 {
 226         struct file *file = vma->vm_file;
 227         struct shm_file_data *sfd = shm_file_data(file);
 228         struct shmid_kernel *shp;
 229
 230         shp = shm_lock(sfd->ns, sfd->id);
 231
 232         if (IS_ERR(shp))
 233                 return PTR_ERR(shp);
 234
 235         if (shp->shm_file != sfd->file) {
 236                 /* ID was reused */
 237                 shm_unlock(shp);
 238                 return -EINVAL;
 239         }
 240
 241         shp->shm_atim = get_seconds();
 242         shp->shm_lprid = task_tgid_vnr(current);
 243         shp->shm_nattch++;
 244         shm_unlock(shp);
 245         return 0;
 246 }
 247
 248 /* This is called by fork, once for every shm attach. */
 249 static void shm_open(struct vm_area_struct *vma)
 250 {
 251         int err = __shm_open(vma);
 252         /*
 253          * We raced in the idr lookup or with shm_destroy().
 254          * Either way, the ID is busted.
 255          */
 256         WARN_ON_ONCE(err);
 257 }
 258
 259 /*
 260  * shm_destroy - free the struct shmid_kernel
 261  *
 262  * @ns: namespace
 263  * @shp: struct to free
 264  *
 265  * It has to be called with shp and shm_ids.rwsem (writer) locked,
 266  * but returns with shp unlocked and freed.
 267  */
 268 static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
 269 {
 270         struct file *shm_file;
 271
 272         shm_file = shp->shm_file;
 273         shp->shm_file = NULL;
 274         ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
 275         shm_rmid(shp);
 276         shm_unlock(shp);
 277         if (!is_file_hugepages(shm_file))
 278                 shmem_lock(shm_file, 0, shp->mlock_user);
 279         else if (shp->mlock_user)
 280                 user_shm_unlock(i_size_read(file_inode(shm_file)),
 281                                 shp->mlock_user);
 282         fput(shm_file);
 283         ipc_rcu_putref(shp, shm_rcu_free);
 284 }
 285
 286 /*
 287  * shm_may_destroy - identifies whether shm segment should be destroyed now
 288  *
 289  * Returns true if and only if there are no active users of the segment and
 290  * one of the following is true:
 291  *
 292  * 1) shmctl(id, IPC_RMID, NULL) was called for this shp
 293  *
 294  * 2) sysctl kernel.shm_rmid_forced is set to 1.
 295  */
 296 static bool shm_may_destroy(struct shmid_kernel *shp)
 297 {
 298         return (shp->shm_nattch == 0) &&
 299                (shp->ns->shm_rmid_forced ||
 300                 (shp->shm_perm.mode & SHM_DEST));
 301 }
 302
 303 /*
 304  * remove the attach descriptor vma.
 305  * free memory for segment if it is marked destroyed.
 306  * The descriptor has already been removed from the current->mm->mmap list
 307  * and will later be kfree()d.
 308  */
 309 static void shm_close(struct vm_area_struct *vma)
 310 {
 311         struct file *file = vma->vm_file;
 312         struct shm_file_data *sfd = shm_file_data(file);
 313         struct shmid_kernel *shp;
 314         struct ipc_namespace *ns = sfd->ns;
 315
 316         down_write(&shm_ids(ns).rwsem);
 317         /* remove from the list of attaches of the shm segment */
 318         shp = shm_lock(ns, sfd->id);
 319
 320         /*
 321          * We raced in the idr lookup or with shm_destroy().
 322          * Either way, the ID is busted.
 323          */
 324         if (WARN_ON_ONCE(IS_ERR(shp)))
 325                 goto done; /* no-op */
 326
 327         shp->shm_lprid = task_tgid_vnr(current);
 328         shp->shm_dtim = get_seconds();
 329         shp->shm_nattch--;
 330         if (shm_may_destroy(shp))
 331                 shm_destroy(ns, shp);
 332         else
 333                 shm_unlock(shp);
 334 done:
 335         up_write(&shm_ids(ns).rwsem);
 336 }
 337
 338 /* Called with ns->shm_ids(ns).rwsem locked */
 339 static int shm_try_destroy_orphaned(int id, void *p, void *data)
 340 {
 341         struct ipc_namespace *ns = data;
 342         struct kern_ipc_perm *ipcp = p;
 343         struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm);
 344
 345         /*
 346          * We want to destroy segments without users and with already
 347          * exit'ed originating process.
 348          *
 349          * As shp->* are changed under rwsem, it's safe to skip shp locking.
 350          */
 351         if (!list_empty(&shp->shm_clist))
 352                 return 0;
 353
 354         if (shm_may_destroy(shp)) {
 355                 shm_lock_by_ptr(shp);
 356                 shm_destroy(ns, shp);
 357         }
 358         return 0;
 359 }
 360
 361 void shm_destroy_orphaned(struct ipc_namespace *ns)
 362 {
 363         down_write(&shm_ids(ns).rwsem);
 364         if (shm_ids(ns).in_use)
 365                 idr_for_each(&shm_ids(ns).ipcs_idr, &shm_try_destroy_orphaned, ns);
 366         up_write(&shm_ids(ns).rwsem);
 367 }
 368
 369 /* Locking assumes this will only be called with task == current */
 370 void exit_shm(struct task_struct *task)
 371 {
 372         for (;;) {
 373                 struct shmid_kernel *shp;
 374                 struct ipc_namespace *ns;
 375
 376                 task_lock(task);
 377
 378                 if (list_empty(&task->sysvshm.shm_clist)) {
 379                         task_unlock(task);
 380                         break;
 381                 }
 382
 383                 shp = list_first_entry(&task->sysvshm.shm_clist, struct shmid_kernel,
 384                                 shm_clist);
 385
 386                 /*
 387                  * 1) Get pointer to the ipc namespace. It is worth to say
 388                  * that this pointer is guaranteed to be valid because
 389                  * shp lifetime is always shorter than namespace lifetime
 390                  * in which shp lives.
 391                  * We taken task_lock it means that shp won't be freed.
 392                  */
 393                 ns = shp->ns;
 394
 395                 /*
 396                  * 2) If kernel.shm_rmid_forced is not set then only keep track of
 397                  * which shmids are orphaned, so that a later set of the sysctl
 398                  * can clean them up.
 399                  */
 400                 if (!ns->shm_rmid_forced)
 401                         goto unlink_continue;
 402
 403                 /*
 404                  * 3) get a reference to the namespace.
 405                  *    The refcount could be already 0. If it is 0, then
 406                  *    the shm objects will be free by free_ipc_work().
 407                  */
 408                 ns = get_ipc_ns_not_zero(ns);
 409                 if (!ns) {
 410 unlink_continue:
 411                         list_del_init(&shp->shm_clist);
 412                         task_unlock(task);
 413                         continue;
 414                 }
 415
 416                 /*
 417                  * 4) get a reference to shp.
 418                  *   This cannot fail: shm_clist_rm() is called before
 419                  *   ipc_rmid(), thus the refcount cannot be 0.
 420                  */
 421                 WARN_ON(!ipc_rcu_getref(&shp->shm_perm));
 422
 423                 /*
 424                  * 5) unlink the shm segment from the list of segments
 425                  *    created by current.
 426                  *    This must be done last. After unlinking,
 427                  *    only the refcounts obtained above prevent IPC_RMID
 428                  *    from destroying the segment or the namespace.
 429                  */
 430                 list_del_init(&shp->shm_clist);
 431
 432                 task_unlock(task);
 433
 434                 /*
 435                  * 6) we have all references
 436                  *    Thus lock & if needed destroy shp.
 437                  */
 438                 down_write(&shm_ids(ns).rwsem);
 439                 shm_lock_by_ptr(shp);
 440                 /*
 441                  * rcu_read_lock was implicitly taken in shm_lock_by_ptr, it's
 442                  * safe to call ipc_rcu_putref here
 443                  */
 444                 ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
 445
 446                 if (ipc_valid_object(&shp->shm_perm)) {
 447                         if (shm_may_destroy(shp))
 448                                 shm_destroy(ns, shp);
 449                         else
 450                                 shm_unlock(shp);
 451                 } else {
 452                         /*
 453                          * Someone else deleted the shp from namespace
 454                          * idr/kht while we have waited.
 455                          * Just unlock and continue.
 456                          */
 457                         shm_unlock(shp);
 458                 }
 459
 460                 up_write(&shm_ids(ns).rwsem);
 461                 put_ipc_ns(ns); /* paired with get_ipc_ns_not_zero */
 462         }
 463 }
 464
 465 static int shm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 466 {
 467         struct file *file = vma->vm_file;
 468         struct shm_file_data *sfd = shm_file_data(file);
 469
 470         return sfd->vm_ops->fault(vma, vmf);
 471 }
 472
 473 static int shm_split(struct vm_area_struct *vma, unsigned long addr)
 474 {
 475         struct file *file = vma->vm_file;
 476         struct shm_file_data *sfd = shm_file_data(file);
 477
 478         if (sfd->vm_ops && sfd->vm_ops->split)
 479                 return sfd->vm_ops->split(vma, addr);
 480
 481         return 0;
 482 }
 483
 484 #ifdef CONFIG_NUMA
 485 static int shm_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
 486 {
 487         struct file *file = vma->vm_file;
 488         struct shm_file_data *sfd = shm_file_data(file);
 489         int err = 0;
 490         if (sfd->vm_ops->set_policy)
 491                 err = sfd->vm_ops->set_policy(vma, new);
 492         return err;
 493 }
 494
 495 static struct mempolicy *shm_get_policy(struct vm_area_struct *vma,
 496                                         unsigned long addr)
 497 {
 498         struct file *file = vma->vm_file;
 499         struct shm_file_data *sfd = shm_file_data(file);
 500         struct mempolicy *pol = NULL;
 501
 502         if (sfd->vm_ops->get_policy)
 503                 pol = sfd->vm_ops->get_policy(vma, addr);
 504         else if (vma->vm_policy)
 505                 pol = vma->vm_policy;
 506
 507         return pol;
 508 }
 509 #endif
 510
 511 static int shm_mmap(struct file *file, struct vm_area_struct *vma)
 512 {
 513         struct shm_file_data *sfd = shm_file_data(file);
 514         int ret;
 515
 516         /*
 517          * In case of remap_file_pages() emulation, the file can represent an
 518          * IPC ID that was removed, and possibly even reused by another shm
 519          * segment already.  Propagate this case as an error to caller.
 520          */
 521         ret =__shm_open(vma);
 522         if (ret)
 523                 return ret;
 524
 525         ret = sfd->file->f_op->mmap(sfd->file, vma);
 526         if (ret) {
 527                 shm_close(vma);
 528                 return ret;
 529         }
 530         sfd->vm_ops = vma->vm_ops;
 531 #ifdef CONFIG_MMU
 532         WARN_ON(!sfd->vm_ops->fault);
 533 #endif
 534         vma->vm_ops = &shm_vm_ops;
 535         return 0;
 536 }
 537
 538 static int shm_release(struct inode *ino, struct file *file)
 539 {
 540         struct shm_file_data *sfd = shm_file_data(file);
 541
 542         put_ipc_ns(sfd->ns);
 543         fput(sfd->file);
 544         shm_file_data(file) = NULL;
 545         kfree(sfd);
 546         return 0;
 547 }
 548
 549 static int shm_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 550 {
 551         struct shm_file_data *sfd = shm_file_data(file);
 552
 553         if (!sfd->file->f_op->fsync)
 554                 return -EINVAL;
 555         return sfd->file->f_op->fsync(sfd->file, start, end, datasync);
 556 }
 557
 558 static long shm_fallocate(struct file *file, int mode, loff_t offset,
 559                           loff_t len)
 560 {
 561         struct shm_file_data *sfd = shm_file_data(file);
 562
 563         if (!sfd->file->f_op->fallocate)
 564                 return -EOPNOTSUPP;
 565         return sfd->file->f_op->fallocate(file, mode, offset, len);
 566 }
 567
 568 static unsigned long shm_get_unmapped_area(struct file *file,
 569         unsigned long addr, unsigned long len, unsigned long pgoff,
 570         unsigned long flags)
 571 {
 572         struct shm_file_data *sfd = shm_file_data(file);
 573         return sfd->file->f_op->get_unmapped_area(sfd->file, addr, len,
 574                                                 pgoff, flags);
 575 }
 576
 577 static const struct file_operations shm_file_operations = {
 578         .mmap           = shm_mmap,
 579         .fsync          = shm_fsync,
 580         .release        = shm_release,
 581         .get_unmapped_area      = shm_get_unmapped_area,
 582         .llseek         = noop_llseek,
 583         .fallocate      = shm_fallocate,
 584 };
 585
 586 /*
 587  * shm_file_operations_huge is now identical to shm_file_operations,
 588  * but we keep it distinct for the sake of is_file_shm_hugepages().
 589  */
 590 static const struct file_operations shm_file_operations_huge = {
 591         .mmap           = shm_mmap,
 592         .fsync          = shm_fsync,
 593         .release        = shm_release,
 594         .get_unmapped_area      = shm_get_unmapped_area,
 595         .llseek         = noop_llseek,
 596         .fallocate      = shm_fallocate,
 597 };
 598
 599 bool is_file_shm_hugepages(struct file *file)
 600 {
 601         return file->f_op == &shm_file_operations_huge;
 602 }
 603
 604 static const struct vm_operations_struct shm_vm_ops = {
 605         .open   = shm_open,     /* callback for a new vm-area open */
 606         .close  = shm_close,    /* callback for when the vm-area is released */
 607         .fault  = shm_fault,
 608         .split  = shm_split,
 609 #if defined(CONFIG_NUMA)
 610         .set_policy = shm_set_policy,
 611         .get_policy = shm_get_policy,
 612 #endif
 613 };
 614
 615 /**
 616  * newseg - Create a new shared memory segment
 617  * @ns: namespace
 618  * @params: ptr to the structure that contains key, size and shmflg
 619  *
 620  * Called with shm_ids.rwsem held as a writer.
 621  */
 622 static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
 623 {
 624         key_t key = params->key;
 625         int shmflg = params->flg;
 626         size_t size = params->u.size;
 627         int error;
 628         struct shmid_kernel *shp;
 629         size_t numpages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
 630         struct file *file;
 631         char name[13];
 632         int id;
 633         vm_flags_t acctflag = 0;
 634
 635         if (size < SHMMIN || size > ns->shm_ctlmax)
 636                 return -EINVAL;
 637
 638         if (numpages << PAGE_SHIFT < size)
 639                 return -ENOSPC;
 640
 641         if (ns->shm_tot + numpages < ns->shm_tot ||
 642                         ns->shm_tot + numpages > ns->shm_ctlall)
 643                 return -ENOSPC;
 644
 645         shp = ipc_rcu_alloc(sizeof(*shp));
 646         if (!shp)
 647                 return -ENOMEM;
 648
 649         shp->shm_perm.key = key;
 650         shp->shm_perm.mode = (shmflg & S_IRWXUGO);
 651         shp->mlock_user = NULL;
 652
 653         shp->shm_perm.security = NULL;
 654         error = security_shm_alloc(shp);
 655         if (error) {
 656                 ipc_rcu_putref(shp, ipc_rcu_free);
 657                 return error;
 658         }
 659
 660         sprintf(name, "SYSV%08x", key);
 661         if (shmflg & SHM_HUGETLB) {
 662                 struct hstate *hs;
 663                 size_t hugesize;
 664
 665                 hs = hstate_sizelog((shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
 666                 if (!hs) {
 667                         error = -EINVAL;
 668                         goto no_file;
 669                 }
 670                 hugesize = ALIGN(size, huge_page_size(hs));
 671
 672                 /* hugetlb_file_setup applies strict accounting */
 673                 if (shmflg & SHM_NORESERVE)
 674                         acctflag = VM_NORESERVE;
 675                 file = hugetlb_file_setup(name, hugesize, acctflag,
 676                                   &shp->mlock_user, HUGETLB_SHMFS_INODE,
 677                                 (shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
 678         } else {
 679                 /*
 680                  * Do not allow no accounting for OVERCOMMIT_NEVER, even
 681                  * if it's asked for.
 682                  */
 683                 if  ((shmflg & SHM_NORESERVE) &&
 684                                 sysctl_overcommit_memory != OVERCOMMIT_NEVER)
 685                         acctflag = VM_NORESERVE;
 686                 file = shmem_kernel_file_setup(name, size, acctflag);
 687         }
 688         error = PTR_ERR(file);
 689         if (IS_ERR(file))
 690                 goto no_file;
 691
 692         shp->shm_cprid = task_tgid_vnr(current);
 693         shp->shm_lprid = 0;
 694         shp->shm_atim = shp->shm_dtim = 0;
 695         shp->shm_ctim = get_seconds();
 696         shp->shm_segsz = size;
 697         shp->shm_nattch = 0;
 698         shp->shm_file = file;
 699         shp->shm_creator = current;
 700
 701         id = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);
 702         if (id < 0) {
 703                 error = id;
 704                 goto no_id;
 705         }
 706
 707         shp->ns = ns;
 708
 709         task_lock(current);
 710         list_add(&shp->shm_clist, &current->sysvshm.shm_clist);
 711         task_unlock(current);
 712
 713         /*
 714          * shmid gets reported as "inode#" in /proc/pid/maps.
 715          * proc-ps tools use this. Changing this will break them.
 716          */
 717         file_inode(file)->i_ino = shp->shm_perm.id;
 718
 719         ns->shm_tot += numpages;
 720         error = shp->shm_perm.id;
 721
 722         ipc_unlock_object(&shp->shm_perm);
 723         rcu_read_unlock();
 724         return error;
 725
 726 no_id:
 727         if (is_file_hugepages(file) && shp->mlock_user)
 728                 user_shm_unlock(size, shp->mlock_user);
 729         fput(file);
 730 no_file:
 731         ipc_rcu_putref(shp, shm_rcu_free);
 732         return error;
 733 }
 734
 735 /*
 736  * Called with shm_ids.rwsem and ipcp locked.
 737  */
 738 static inline int shm_security(struct kern_ipc_perm *ipcp, int shmflg)
 739 {
 740         struct shmid_kernel *shp;
 741
 742         shp = container_of(ipcp, struct shmid_kernel, shm_perm);
 743         return security_shm_associate(shp, shmflg);
 744 }
 745
 746 /*
 747  * Called with shm_ids.rwsem and ipcp locked.
 748  */
 749 static inline int shm_more_checks(struct kern_ipc_perm *ipcp,
 750                                 struct ipc_params *params)
 751 {
 752         struct shmid_kernel *shp;
 753
 754         shp = container_of(ipcp, struct shmid_kernel, shm_perm);
 755         if (shp->shm_segsz < params->u.size)
 756                 return -EINVAL;
 757
 758         return 0;
 759 }
 760
 761 SYSCALL_DEFINE3(shmget, key_t, key, size_t, size, int, shmflg)
 762 {
 763         struct ipc_namespace *ns;
 764         static const struct ipc_ops shm_ops = {
 765                 .getnew = newseg,
 766                 .associate = shm_security,
 767                 .more_checks = shm_more_checks,
 768         };
 769         struct ipc_params shm_params;
 770
 771         ns = current->nsproxy->ipc_ns;
 772
 773         shm_params.key = key;
 774         shm_params.flg = shmflg;
 775         shm_params.u.size = size;
 776
 777         return ipcget(ns, &shm_ids(ns), &shm_ops, &shm_params);
 778 }
 779
 780 static inline unsigned long copy_shmid_to_user(void __user *buf, struct shmid64_ds *in, int version)
 781 {
 782         switch (version) {
 783         case IPC_64:
 784                 return copy_to_user(buf, in, sizeof(*in));
 785         case IPC_OLD:
 786             {
 787                 struct shmid_ds out;
 788
 789                 memset(&out, 0, sizeof(out));
 790                 ipc64_perm_to_ipc_perm(&in->shm_perm, &out.shm_perm);
 791                 out.shm_segsz   = in->shm_segsz;
 792                 out.shm_atime   = in->shm_atime;
 793                 out.shm_dtime   = in->shm_dtime;
 794                 out.shm_ctime   = in->shm_ctime;
 795                 out.shm_cpid    = in->shm_cpid;
 796                 out.shm_lpid    = in->shm_lpid;
 797                 out.shm_nattch  = in->shm_nattch;
 798
 799                 return copy_to_user(buf, &out, sizeof(out));
 800             }
 801         default:
 802                 return -EINVAL;
 803         }
 804 }
 805
 806 static inline unsigned long
 807 copy_shmid_from_user(struct shmid64_ds *out, void __user *buf, int version)
 808 {
 809         switch (version) {
 810         case IPC_64:
 811                 if (copy_from_user(out, buf, sizeof(*out)))
 812                         return -EFAULT;
 813                 return 0;
 814         case IPC_OLD:
 815             {
 816                 struct shmid_ds tbuf_old;
 817
 818                 if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
 819                         return -EFAULT;
 820
 821                 out->shm_perm.uid       = tbuf_old.shm_perm.uid;
 822                 out->shm_perm.gid       = tbuf_old.shm_perm.gid;
 823                 out->shm_perm.mode      = tbuf_old.shm_perm.mode;
 824
 825                 return 0;
 826             }
 827         default:
 828                 return -EINVAL;
 829         }
 830 }
 831
 832 static inline unsigned long copy_shminfo_to_user(void __user *buf, struct shminfo64 *in, int version)
 833 {
 834         switch (version) {
 835         case IPC_64:
 836                 return copy_to_user(buf, in, sizeof(*in));
 837         case IPC_OLD:
 838             {
 839                 struct shminfo out;
 840
 841                 if (in->shmmax > INT_MAX)
 842                         out.shmmax = INT_MAX;
 843                 else
 844                         out.shmmax = (int)in->shmmax;
 845
 846                 out.shmmin      = in->shmmin;
 847                 out.shmmni      = in->shmmni;
 848                 out.shmseg      = in->shmseg;
 849                 out.shmall      = in->shmall;
 850
 851                 return copy_to_user(buf, &out, sizeof(out));
 852             }
 853         default:
 854                 return -EINVAL;
 855         }
 856 }
 857
 858 /*
 859  * Calculate and add used RSS and swap pages of a shm.
 860  * Called with shm_ids.rwsem held as a reader
 861  */
 862 static void shm_add_rss_swap(struct shmid_kernel *shp,
 863         unsigned long *rss_add, unsigned long *swp_add)
 864 {
 865         struct inode *inode;
 866
 867         inode = file_inode(shp->shm_file);
 868
 869         if (is_file_hugepages(shp->shm_file)) {
 870                 struct address_space *mapping = inode->i_mapping;
 871                 struct hstate *h = hstate_file(shp->shm_file);
 872                 *rss_add += pages_per_huge_page(h) * mapping->nrpages;
 873         } else {
 874 #ifdef CONFIG_SHMEM
 875                 struct shmem_inode_info *info = SHMEM_I(inode);
 876                 spin_lock_irq(&info->lock);
 877                 *rss_add += inode->i_mapping->nrpages;
 878                 *swp_add += info->swapped;
 879                 spin_unlock_irq(&info->lock);
 880 #else
 881                 *rss_add += inode->i_mapping->nrpages;
 882 #endif
 883         }
 884 }
 885
 886 /*
 887  * Called with shm_ids.rwsem held as a reader
 888  */
 889 static void shm_get_stat(struct ipc_namespace *ns, unsigned long *rss,
 890                 unsigned long *swp)
 891 {
 892         int next_id;
 893         int total, in_use;
 894
 895         *rss = 0;
 896         *swp = 0;
 897
 898         in_use = shm_ids(ns).in_use;
 899
 900         for (total = 0, next_id = 0; total < in_use; next_id++) {
 901                 struct kern_ipc_perm *ipc;
 902                 struct shmid_kernel *shp;
 903
 904                 ipc = idr_find(&shm_ids(ns).ipcs_idr, next_id);
 905                 if (ipc == NULL)
 906                         continue;
 907                 shp = container_of(ipc, struct shmid_kernel, shm_perm);
 908
 909                 shm_add_rss_swap(shp, rss, swp);
 910
 911                 total++;
 912         }
 913 }
 914
 915 /*
 916  * This function handles some shmctl commands which require the rwsem
 917  * to be held in write mode.
 918  * NOTE: no locks must be held, the rwsem is taken inside this function.
 919  */
 920 static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
 921                        struct shmid_ds __user *buf, int version)
 922 {
 923         struct kern_ipc_perm *ipcp;
 924         struct shmid64_ds shmid64;
 925         struct shmid_kernel *shp;
 926         int err;
 927
 928         if (cmd == IPC_SET) {
 929                 if (copy_shmid_from_user(&shmid64, buf, version))
 930                         return -EFAULT;
 931         }
 932
 933         down_write(&shm_ids(ns).rwsem);
 934         rcu_read_lock();
 935
 936         ipcp = ipcctl_pre_down_nolock(ns, &shm_ids(ns), shmid, cmd,
 937                                       &shmid64.shm_perm, 0);
 938         if (IS_ERR(ipcp)) {
 939                 err = PTR_ERR(ipcp);
 940                 goto out_unlock1;
 941         }
 942
 943         shp = container_of(ipcp, struct shmid_kernel, shm_perm);
 944
 945         err = security_shm_shmctl(shp, cmd);
 946         if (err)
 947                 goto out_unlock1;
 948
 949         switch (cmd) {
 950         case IPC_RMID:
 951                 ipc_lock_object(&shp->shm_perm);
 952                 /* do_shm_rmid unlocks the ipc object and rcu */
 953                 do_shm_rmid(ns, ipcp);
 954                 goto out_up;
 955         case IPC_SET:
 956                 ipc_lock_object(&shp->shm_perm);
 957                 err = ipc_update_perm(&shmid64.shm_perm, ipcp);
 958                 if (err)
 959                         goto out_unlock0;
 960                 shp->shm_ctim = get_seconds();
 961                 break;
 962         default:
 963                 err = -EINVAL;
 964                 goto out_unlock1;
 965         }
 966
 967 out_unlock0:
 968         ipc_unlock_object(&shp->shm_perm);
 969 out_unlock1:
 970         rcu_read_unlock();
 971 out_up:
 972         up_write(&shm_ids(ns).rwsem);
 973         return err;
 974 }
 975
 976 static int shmctl_nolock(struct ipc_namespace *ns, int shmid,
 977                          int cmd, int version, void __user *buf)
 978 {
 979         int err;
 980         struct shmid_kernel *shp;
 981
 982         /* preliminary security checks for *_INFO */
 983         if (cmd == IPC_INFO || cmd == SHM_INFO) {
 984                 err = security_shm_shmctl(NULL, cmd);
 985                 if (err)
 986                         return err;
 987         }
 988
 989         switch (cmd) {
 990         case IPC_INFO:
 991         {
 992                 struct shminfo64 shminfo;
 993
 994                 memset(&shminfo, 0, sizeof(shminfo));
 995                 shminfo.shmmni = shminfo.shmseg = ns->shm_ctlmni;
 996                 shminfo.shmmax = ns->shm_ctlmax;
 997                 shminfo.shmall = ns->shm_ctlall;
 998
 999                 shminfo.shmmin = SHMMIN;
1000                 if (copy_shminfo_to_user(buf, &shminfo, version))
1001                         return -EFAULT;
1002
1003                 down_read(&shm_ids(ns).rwsem);
1004                 err = ipc_get_maxid(&shm_ids(ns));
1005                 up_read(&shm_ids(ns).rwsem);
1006
1007                 if (err < 0)
1008                         err = 0;
1009                 goto out;
1010         }
1011         case SHM_INFO:
1012         {
1013                 struct shm_info shm_info;
1014
1015                 memset(&shm_info, 0, sizeof(shm_info));
1016                 down_read(&shm_ids(ns).rwsem);
1017                 shm_info.used_ids = shm_ids(ns).in_use;
1018                 shm_get_stat(ns, &shm_info.shm_rss, &shm_info.shm_swp);
1019                 shm_info.shm_tot = ns->shm_tot;
1020                 shm_info.swap_attempts = 0;
1021                 shm_info.swap_successes = 0;
1022                 err = ipc_get_maxid(&shm_ids(ns));
1023                 up_read(&shm_ids(ns).rwsem);
1024                 if (copy_to_user(buf, &shm_info, sizeof(shm_info))) {
1025                         err = -EFAULT;
1026                         goto out;
1027                 }
1028
1029                 err = err < 0 ? 0 : err;
1030                 goto out;
1031         }
1032         case SHM_STAT:
1033         case IPC_STAT:
1034         {
1035                 struct shmid64_ds tbuf;
1036                 int result;
1037
1038                 rcu_read_lock();
1039                 if (cmd == SHM_STAT) {
1040                         shp = shm_obtain_object(ns, shmid);
1041                         if (IS_ERR(shp)) {
1042                                 err = PTR_ERR(shp);
1043                                 goto out_unlock;
1044                         }
1045                         result = shp->shm_perm.id;
1046                 } else {
1047                         shp = shm_obtain_object_check(ns, shmid);
1048                         if (IS_ERR(shp)) {
1049                                 err = PTR_ERR(shp);
1050                                 goto out_unlock;
1051                         }
1052                         result = 0;
1053                 }
1054
1055                 err = -EACCES;
1056                 if (ipcperms(ns, &shp->shm_perm, S_IRUGO))
1057                         goto out_unlock;
1058
1059                 err = security_shm_shmctl(shp, cmd);
1060                 if (err)
1061                         goto out_unlock;
1062
1063                 memset(&tbuf, 0, sizeof(tbuf));
1064                 kernel_to_ipc64_perm(&shp->shm_perm, &tbuf.shm_perm);
1065                 tbuf.shm_segsz  = shp->shm_segsz;
1066                 tbuf.shm_atime  = shp->shm_atim;
1067                 tbuf.shm_dtime  = shp->shm_dtim;
1068                 tbuf.shm_ctime  = shp->shm_ctim;
1069                 tbuf.shm_cpid   = shp->shm_cprid;
1070                 tbuf.shm_lpid   = shp->shm_lprid;
1071                 tbuf.shm_nattch = shp->shm_nattch;
1072                 rcu_read_unlock();
1073
1074                 if (copy_shmid_to_user(buf, &tbuf, version))
1075                         err = -EFAULT;
1076                 else
1077                         err = result;
1078                 goto out;
1079         }
1080         default:
1081                 return -EINVAL;
1082         }
1083
1084 out_unlock:
1085         rcu_read_unlock();
1086 out:
1087         return err;
1088 }
1089
1090 SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
1091 {
1092         struct shmid_kernel *shp;
1093         int err, version;
1094         struct ipc_namespace *ns;
1095
1096         if (cmd < 0 || shmid < 0)
1097                 return -EINVAL;
1098
1099         version = ipc_parse_version(&cmd);
1100         ns = current->nsproxy->ipc_ns;
1101
1102         switch (cmd) {
1103         case IPC_INFO:
1104         case SHM_INFO:
1105         case SHM_STAT:
1106         case IPC_STAT:
1107                 return shmctl_nolock(ns, shmid, cmd, version, buf);
1108         case IPC_RMID:
1109         case IPC_SET:
1110                 return shmctl_down(ns, shmid, cmd, buf, version);
1111         case SHM_LOCK:
1112         case SHM_UNLOCK:
1113         {
1114                 struct file *shm_file;
1115
1116                 rcu_read_lock();
1117                 shp = shm_obtain_object_check(ns, shmid);
1118                 if (IS_ERR(shp)) {
1119                         err = PTR_ERR(shp);
1120                         goto out_unlock1;
1121                 }
1122
1123                 audit_ipc_obj(&(shp->shm_perm));
1124                 err = security_shm_shmctl(shp, cmd);
1125                 if (err)
1126                         goto out_unlock1;
1127
1128                 ipc_lock_object(&shp->shm_perm);
1129
1130                 /* check if shm_destroy() is tearing down shp */
1131                 if (!ipc_valid_object(&shp->shm_perm)) {
1132                         err = -EIDRM;
1133                         goto out_unlock0;
1134                 }
1135
1136                 if (!ns_capable(ns->user_ns, CAP_IPC_LOCK)) {
1137                         kuid_t euid = current_euid();
1138                         if (!uid_eq(euid, shp->shm_perm.uid) &&
1139                             !uid_eq(euid, shp->shm_perm.cuid)) {
1140                                 err = -EPERM;
1141                                 goto out_unlock0;
1142                         }
1143                         if (cmd == SHM_LOCK && !rlimit(RLIMIT_MEMLOCK)) {
1144                                 err = -EPERM;
1145                                 goto out_unlock0;
1146                         }
1147                 }
1148
1149                 shm_file = shp->shm_file;
1150                 if (is_file_hugepages(shm_file))
1151                         goto out_unlock0;
1152
1153                 if (cmd == SHM_LOCK) {
1154                         struct user_struct *user = current_user();
1155                         err = shmem_lock(shm_file, 1, user);
1156                         if (!err && !(shp->shm_perm.mode & SHM_LOCKED)) {
1157                                 shp->shm_perm.mode |= SHM_LOCKED;
1158                                 shp->mlock_user = user;
1159                         }
1160                         goto out_unlock0;
1161                 }
1162
1163                 /* SHM_UNLOCK */
1164                 if (!(shp->shm_perm.mode & SHM_LOCKED))
1165                         goto out_unlock0;
1166                 shmem_lock(shm_file, 0, shp->mlock_user);
1167                 shp->shm_perm.mode &= ~SHM_LOCKED;
1168                 shp->mlock_user = NULL;
1169                 get_file(shm_file);
1170                 ipc_unlock_object(&shp->shm_perm);
1171                 rcu_read_unlock();
1172                 shmem_unlock_mapping(shm_file->f_mapping);
1173
1174                 fput(shm_file);
1175                 return err;
1176         }
1177         default:
1178                 return -EINVAL;
1179         }
1180
1181 out_unlock0:
1182         ipc_unlock_object(&shp->shm_perm);
1183 out_unlock1:
1184         rcu_read_unlock();
1185         return err;
1186 }
1187
1188 /*
1189  * Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists.
1190  *
1191  * NOTE! Despite the name, this is NOT a direct system call entrypoint. The
1192  * "raddr" thing points to kernel space, and there has to be a wrapper around
1193  * this.
1194  */
1195 long do_shmat(int shmid, char __user *shmaddr, int shmflg,
1196               ulong *raddr, unsigned long shmlba)
1197 {
1198         struct shmid_kernel *shp;
1199         unsigned long addr;
1200         unsigned long size;
1201         struct file *file;
1202         int    err;
1203         unsigned long flags;
1204         unsigned long prot;
1205         int acc_mode;
1206         struct ipc_namespace *ns;
1207         struct shm_file_data *sfd;
1208         struct path path;
1209         fmode_t f_mode;
1210         unsigned long populate = 0;
1211
1212         err = -EINVAL;
1213         if (shmid < 0)
1214                 goto out;
1215         else if ((addr = (ulong)shmaddr)) {
1216                 if (addr & (shmlba - 1)) {
1217                         if (shmflg & SHM_RND) {
1218                                 addr &= ~(shmlba - 1);  /* round down */
1219
1220                                 /*
1221                                  * Ensure that the round-down is non-nil
1222                                  * when remapping. This can happen for
1223                                  * cases when addr < shmlba.
1224                                  */
1225                                 if (!addr && (shmflg & SHM_REMAP))
1226                                         goto out;
1227                         } else
1228 #ifndef __ARCH_FORCE_SHMLBA
1229                                 if (addr & ~PAGE_MASK)
1230 #endif
1231                                         goto out;
1232                 }
1233                 flags = MAP_SHARED | MAP_FIXED;
1234         } else {
1235                 if ((shmflg & SHM_REMAP))
1236                         goto out;
1237
1238                 flags = MAP_SHARED;
1239         }
1240
1241         if (shmflg & SHM_RDONLY) {
1242                 prot = PROT_READ;
1243                 acc_mode = S_IRUGO;
1244                 f_mode = FMODE_READ;
1245         } else {
1246                 prot = PROT_READ | PROT_WRITE;
1247                 acc_mode = S_IRUGO | S_IWUGO;
1248                 f_mode = FMODE_READ | FMODE_WRITE;
1249         }
1250         if (shmflg & SHM_EXEC) {
1251                 prot |= PROT_EXEC;
1252                 acc_mode |= S_IXUGO;
1253         }
1254
1255         /*
1256          * We cannot rely on the fs check since SYSV IPC does have an
1257          * additional creator id...
1258          */
1259         ns = current->nsproxy->ipc_ns;
1260         rcu_read_lock();
1261         shp = shm_obtain_object_check(ns, shmid);
1262         if (IS_ERR(shp)) {
1263                 err = PTR_ERR(shp);
1264                 goto out_unlock;
1265         }
1266
1267         err = -EACCES;
1268         if (ipcperms(ns, &shp->shm_perm, acc_mode))
1269                 goto out_unlock;
1270
1271         err = security_shm_shmat(shp, shmaddr, shmflg);
1272         if (err)
1273                 goto out_unlock;
1274
1275         ipc_lock_object(&shp->shm_perm);
1276
1277         /* check if shm_destroy() is tearing down shp */
1278         if (!ipc_valid_object(&shp->shm_perm)) {
1279                 ipc_unlock_object(&shp->shm_perm);
1280                 err = -EIDRM;
1281                 goto out_unlock;
1282         }
1283
1284         path = shp->shm_file->f_path;
1285         path_get(&path);
1286         shp->shm_nattch++;
1287         size = i_size_read(d_inode(path.dentry));
1288         ipc_unlock_object(&shp->shm_perm);
1289         rcu_read_unlock();
1290
1291         err = -ENOMEM;
1292         sfd = kzalloc(sizeof(*sfd), GFP_KERNEL);
1293         if (!sfd) {
1294                 path_put(&path);
1295                 goto out_nattch;
1296         }
1297
1298         file = alloc_file(&path, f_mode,
1299                           is_file_hugepages(shp->shm_file) ?
1300                                 &shm_file_operations_huge :
1301                                 &shm_file_operations);
1302         err = PTR_ERR(file);
1303         if (IS_ERR(file)) {
1304                 kfree(sfd);
1305                 path_put(&path);
1306                 goto out_nattch;
1307         }
1308
1309         file->private_data = sfd;
1310         file->f_mapping = shp->shm_file->f_mapping;
1311         sfd->id = shp->shm_perm.id;
1312         sfd->ns = get_ipc_ns(ns);
1313         /*
1314          * We need to take a reference to the real shm file to prevent the
1315          * pointer from becoming stale in cases where the lifetime of the outer
1316          * file extends beyond that of the shm segment.  It's not usually
1317          * possible, but it can happen during remap_file_pages() emulation as
1318          * that unmaps the memory, then does ->mmap() via file reference only.
1319          * We'll deny the ->mmap() if the shm segment was since removed, but to
1320          * detect shm ID reuse we need to compare the file pointers.
1321          */
1322         sfd->file = get_file(shp->shm_file);
1323         sfd->vm_ops = NULL;
1324
1325         err = security_mmap_file(file, prot, flags);
1326         if (err)
1327                 goto out_fput;
1328
1329         if (down_write_killable(&current->mm->mmap_sem)) {
1330                 err = -EINTR;
1331                 goto out_fput;
1332         }
1333
1334         if (addr && !(shmflg & SHM_REMAP)) {
1335                 err = -EINVAL;
1336                 if (addr + size < addr)
1337                         goto invalid;
1338
1339                 if (find_vma_intersection(current->mm, addr, addr + size))
1340                         goto invalid;
1341         }
1342
1343         addr = do_mmap_pgoff(file, addr, size, prot, flags, 0, &populate);
1344         *raddr = addr;
1345         err = 0;
1346         if (IS_ERR_VALUE(addr))
1347                 err = (long)addr;
1348 invalid:
1349         up_write(&current->mm->mmap_sem);
1350         if (populate)
1351                 mm_populate(addr, populate);
1352
1353 out_fput:
1354         fput(file);
1355
1356 out_nattch:
1357         down_write(&shm_ids(ns).rwsem);
1358         shp = shm_lock(ns, shmid);
1359         shp->shm_nattch--;
1360
1361         if (shm_may_destroy(shp))
1362                 shm_destroy(ns, shp);
1363         else
1364                 shm_unlock(shp);
1365         up_write(&shm_ids(ns).rwsem);
1366         return err;
1367
1368 out_unlock:
1369         rcu_read_unlock();
1370 out:
1371         return err;
1372 }
1373
1374 SYSCALL_DEFINE3(shmat, int, shmid, char __user *, shmaddr, int, shmflg)
1375 {
1376         unsigned long ret;
1377         long err;
1378
1379         err = do_shmat(shmid, shmaddr, shmflg, &ret, SHMLBA);
1380         if (err)
1381                 return err;
1382         force_successful_syscall_return();
1383         return (long)ret;
1384 }
1385
1386 /*
1387  * detach and kill segment if marked destroyed.
1388  * The work is done in shm_close.
1389  */
1390 SYSCALL_DEFINE1(shmdt, char __user *, shmaddr)
1391 {
1392         struct mm_struct *mm = current->mm;
1393         struct vm_area_struct *vma;
1394         unsigned long addr = (unsigned long)shmaddr;
1395         int retval = -EINVAL;
1396 #ifdef CONFIG_MMU
1397         loff_t size = 0;
1398         struct file *file;
1399         struct vm_area_struct *next;
1400 #endif
1401
1402         if (addr & ~PAGE_MASK)
1403                 return retval;
1404
1405         if (down_write_killable(&mm->mmap_sem))
1406                 return -EINTR;
1407
1408         /*
1409          * This function tries to be smart and unmap shm segments that
1410          * were modified by partial mlock or munmap calls:
1411          * - It first determines the size of the shm segment that should be
1412          *   unmapped: It searches for a vma that is backed by shm and that
1413          *   started at address shmaddr. It records it's size and then unmaps
1414          *   it.
1415          * - Then it unmaps all shm vmas that started at shmaddr and that
1416          *   are within the initially determined size and that are from the
1417          *   same shm segment from which we determined the size.
1418          * Errors from do_munmap are ignored: the function only fails if
1419          * it's called with invalid parameters or if it's called to unmap
1420          * a part of a vma. Both calls in this function are for full vmas,
1421          * the parameters are directly copied from the vma itself and always
1422          * valid - therefore do_munmap cannot fail. (famous last words?)
1423          */
1424         /*
1425          * If it had been mremap()'d, the starting address would not
1426          * match the usual checks anyway. So assume all vma's are
1427          * above the starting address given.
1428          */
1429         vma = find_vma(mm, addr);
1430
1431 #ifdef CONFIG_MMU
1432         while (vma) {
1433                 next = vma->vm_next;
1434
1435                 /*
1436                  * Check if the starting address would match, i.e. it's
1437                  * a fragment created by mprotect() and/or munmap(), or it
1438                  * otherwise it starts at this address with no hassles.
1439                  */
1440                 if ((vma->vm_ops == &shm_vm_ops) &&
1441                         (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) {
1442
1443                         /*
1444                          * Record the file of the shm segment being
1445                          * unmapped.  With mremap(), someone could place
1446                          * page from another segment but with equal offsets
1447                          * in the range we are unmapping.
1448                          */
1449                         file = vma->vm_file;
1450                         size = i_size_read(file_inode(vma->vm_file));
1451                         do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start);
1452                         /*
1453                          * We discovered the size of the shm segment, so
1454                          * break out of here and fall through to the next
1455                          * loop that uses the size information to stop
1456                          * searching for matching vma's.
1457                          */
1458                         retval = 0;
1459                         vma = next;
1460                         break;
1461                 }
1462                 vma = next;
1463         }
1464
1465         /*
1466          * We need look no further than the maximum address a fragment
1467          * could possibly have landed at. Also cast things to loff_t to
1468          * prevent overflows and make comparisons vs. equal-width types.
1469          */
1470         size = PAGE_ALIGN(size);
1471         while (vma && (loff_t)(vma->vm_end - addr) <= size) {
1472                 next = vma->vm_next;
1473
1474                 /* finding a matching vma now does not alter retval */
1475                 if ((vma->vm_ops == &shm_vm_ops) &&
1476                     ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) &&
1477                     (vma->vm_file == file))
1478                         do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start);
1479                 vma = next;
1480         }
1481
1482 #else /* CONFIG_MMU */
1483         /* under NOMMU conditions, the exact address to be destroyed must be
1484          * given */
1485         if (vma && vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) {
1486                 do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start);
1487                 retval = 0;
1488         }
1489
1490 #endif
1491
1492         up_write(&mm->mmap_sem);
1493         return retval;
1494 }
1495
1496 #ifdef CONFIG_PROC_FS
1497 static int sysvipc_shm_proc_show(struct seq_file *s, void *it)
1498 {
1499         struct user_namespace *user_ns = seq_user_ns(s);
1500         struct shmid_kernel *shp = it;
1501         unsigned long rss = 0, swp = 0;
1502
1503         shm_add_rss_swap(shp, &rss, &swp);
1504
1505 #if BITS_PER_LONG <= 32
1506 #define SIZE_SPEC "%10lu"
1507 #else
1508 #define SIZE_SPEC "%21lu"
1509 #endif
1510
1511         seq_printf(s,
1512                    "%10d %10d  %4o " SIZE_SPEC " %5u %5u  "
1513                    "%5lu %5u %5u %5u %5u %10lu %10lu %10lu "
1514                    SIZE_SPEC " " SIZE_SPEC "\n",
1515                    shp->shm_perm.key,
1516                    shp->shm_perm.id,
1517                    shp->shm_perm.mode,
1518                    shp->shm_segsz,
1519                    shp->shm_cprid,
1520                    shp->shm_lprid,
1521                    shp->shm_nattch,
1522                    from_kuid_munged(user_ns, shp->shm_perm.uid),
1523                    from_kgid_munged(user_ns, shp->shm_perm.gid),
1524                    from_kuid_munged(user_ns, shp->shm_perm.cuid),
1525                    from_kgid_munged(user_ns, shp->shm_perm.cgid),
1526                    shp->shm_atim,
1527                    shp->shm_dtim,
1528                    shp->shm_ctim,
1529                    rss * PAGE_SIZE,
1530                    swp * PAGE_SIZE);
1531
1532         return 0;
1533 }
1534 #endif