drivers/staging/android/ion/ion.c

   1 /*
   2  *
   3  * drivers/staging/android/ion/ion.c
   4  *
   5  * Copyright (C) 2011 Google, Inc.
   6  *
   7  * This software is licensed under the terms of the GNU General Public
   8  * License version 2, as published by the Free Software Foundation, and
   9  * may be copied, distributed, and modified under those terms.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  */
  17
  18 #include <linux/device.h>
  19 #include <linux/err.h>
  20 #include <linux/file.h>
  21 #include <linux/freezer.h>
  22 #include <linux/fs.h>
  23 #include <linux/anon_inodes.h>
  24 #include <linux/kthread.h>
  25 #include <linux/list.h>
  26 #include <linux/memblock.h>
  27 #include <linux/miscdevice.h>
  28 #include <linux/export.h>
  29 #include <linux/mm.h>
  30 #include <linux/mm_types.h>
  31 #include <linux/rbtree.h>
  32 #include <linux/slab.h>
  33 #include <linux/seq_file.h>
  34 #include <linux/uaccess.h>
  35 #include <linux/vmalloc.h>
  36 #include <linux/debugfs.h>
  37 #include <linux/dma-buf.h>
  38 #include <linux/idr.h>
  39 #include <linux/sched/task.h>
  40
  41 #include "ion.h"
  42
  43 static struct ion_device *internal_dev;
  44 static int heap_id;
  45
  46 bool ion_buffer_cached(struct ion_buffer *buffer)
  47 {
  48         return !!(buffer->flags & ION_FLAG_CACHED);
  49 }
  50
  51 /* this function should only be called while dev->lock is held */
  52 static void ion_buffer_add(struct ion_device *dev,
  53                            struct ion_buffer *buffer)
  54 {
  55         struct rb_node **p = &dev->buffers.rb_node;
  56         struct rb_node *parent = NULL;
  57         struct ion_buffer *entry;
  58
  59         while (*p) {
  60                 parent = *p;
  61                 entry = rb_entry(parent, struct ion_buffer, node);
  62
  63                 if (buffer < entry) {
  64                         p = &(*p)->rb_left;
  65                 } else if (buffer > entry) {
  66                         p = &(*p)->rb_right;
  67                 } else {
  68                         pr_err("%s: buffer already found.", __func__);
  69                         BUG();
  70                 }
  71         }
  72
  73         rb_link_node(&buffer->node, parent, p);
  74         rb_insert_color(&buffer->node, &dev->buffers);
  75 }
  76
  77 /* this function should only be called while dev->lock is held */
  78 static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
  79                                             struct ion_device *dev,
  80                                             unsigned long len,
  81                                             unsigned long flags)
  82 {
  83         struct ion_buffer *buffer;
  84         struct sg_table *table;
  85         int ret;
  86
  87         buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
  88         if (!buffer)
  89                 return ERR_PTR(-ENOMEM);
  90
  91         buffer->heap = heap;
  92         buffer->flags = flags;
  93
  94         ret = heap->ops->allocate(heap, buffer, len, flags);
  95
  96         if (ret) {
  97                 if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE))
  98                         goto err2;
  99
 100                 ion_heap_freelist_drain(heap, 0);
 101                 ret = heap->ops->allocate(heap, buffer, len, flags);
 102                 if (ret)
 103                         goto err2;
 104         }
 105
 106         if (!buffer->sg_table) {
 107                 WARN_ONCE(1, "This heap needs to set the sgtable");
 108                 ret = -EINVAL;
 109                 goto err1;
 110         }
 111
 112         table = buffer->sg_table;
 113         buffer->dev = dev;
 114         buffer->size = len;
 115
 116         buffer->dev = dev;
 117         buffer->size = len;
 118         INIT_LIST_HEAD(&buffer->attachments);
 119         mutex_init(&buffer->lock);
 120         mutex_lock(&dev->buffer_lock);
 121         ion_buffer_add(dev, buffer);
 122         mutex_unlock(&dev->buffer_lock);
 123         return buffer;
 124
 125 err1:
 126         heap->ops->free(buffer);
 127 err2:
 128         kfree(buffer);
 129         return ERR_PTR(ret);
 130 }
 131
 132 void ion_buffer_destroy(struct ion_buffer *buffer)
 133 {
 134         if (buffer->kmap_cnt > 0) {
 135                 pr_warn_once("%s: buffer still mapped in the kernel\n",
 136                              __func__);
 137                 buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
 138         }
 139         buffer->heap->ops->free(buffer);
 140         kfree(buffer);
 141 }
 142
 143 static void _ion_buffer_destroy(struct ion_buffer *buffer)
 144 {
 145         struct ion_heap *heap = buffer->heap;
 146         struct ion_device *dev = buffer->dev;
 147
 148         mutex_lock(&dev->buffer_lock);
 149         rb_erase(&buffer->node, &dev->buffers);
 150         mutex_unlock(&dev->buffer_lock);
 151
 152         if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
 153                 ion_heap_freelist_add(heap, buffer);
 154         else
 155                 ion_buffer_destroy(buffer);
 156 }
 157
 158 static void *ion_buffer_kmap_get(struct ion_buffer *buffer)
 159 {
 160         void *vaddr;
 161
 162         if (buffer->kmap_cnt) {
 163                 buffer->kmap_cnt++;
 164                 return buffer->vaddr;
 165         }
 166         vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer);
 167         if (WARN_ONCE(!vaddr,
 168                       "heap->ops->map_kernel should return ERR_PTR on error"))
 169                 return ERR_PTR(-EINVAL);
 170         if (IS_ERR(vaddr))
 171                 return vaddr;
 172         buffer->vaddr = vaddr;
 173         buffer->kmap_cnt++;
 174         return vaddr;
 175 }
 176
 177 static void ion_buffer_kmap_put(struct ion_buffer *buffer)
 178 {
 179         buffer->kmap_cnt--;
 180         if (!buffer->kmap_cnt) {
 181                 buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
 182                 buffer->vaddr = NULL;
 183         }
 184 }
 185
 186 static struct sg_table *dup_sg_table(struct sg_table *table)
 187 {
 188         struct sg_table *new_table;
 189         int ret, i;
 190         struct scatterlist *sg, *new_sg;
 191
 192         new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
 193         if (!new_table)
 194                 return ERR_PTR(-ENOMEM);
 195
 196         ret = sg_alloc_table(new_table, table->nents, GFP_KERNEL);
 197         if (ret) {
 198                 kfree(new_table);
 199                 return ERR_PTR(-ENOMEM);
 200         }
 201
 202         new_sg = new_table->sgl;
 203         for_each_sg(table->sgl, sg, table->nents, i) {
 204                 memcpy(new_sg, sg, sizeof(*sg));
 205                 sg->dma_address = 0;
 206                 new_sg = sg_next(new_sg);
 207         }
 208
 209         return new_table;
 210 }
 211
 212 static void free_duped_table(struct sg_table *table)
 213 {
 214         sg_free_table(table);
 215         kfree(table);
 216 }
 217
 218 struct ion_dma_buf_attachment {
 219         struct device *dev;
 220         struct sg_table *table;
 221         struct list_head list;
 222 };
 223
 224 static int ion_dma_buf_attach(struct dma_buf *dmabuf, struct device *dev,
 225                               struct dma_buf_attachment *attachment)
 226 {
 227         struct ion_dma_buf_attachment *a;
 228         struct sg_table *table;
 229         struct ion_buffer *buffer = dmabuf->priv;
 230
 231         a = kzalloc(sizeof(*a), GFP_KERNEL);
 232         if (!a)
 233                 return -ENOMEM;
 234
 235         table = dup_sg_table(buffer->sg_table);
 236         if (IS_ERR(table)) {
 237                 kfree(a);
 238                 return -ENOMEM;
 239         }
 240
 241         a->table = table;
 242         a->dev = dev;
 243         INIT_LIST_HEAD(&a->list);
 244
 245         attachment->priv = a;
 246
 247         mutex_lock(&buffer->lock);
 248         list_add(&a->list, &buffer->attachments);
 249         mutex_unlock(&buffer->lock);
 250
 251         return 0;
 252 }
 253
 254 static void ion_dma_buf_detatch(struct dma_buf *dmabuf,
 255                                 struct dma_buf_attachment *attachment)
 256 {
 257         struct ion_dma_buf_attachment *a = attachment->priv;
 258         struct ion_buffer *buffer = dmabuf->priv;
 259
 260         mutex_lock(&buffer->lock);
 261         list_del(&a->list);
 262         mutex_unlock(&buffer->lock);
 263         free_duped_table(a->table);
 264
 265         kfree(a);
 266 }
 267
 268 static struct sg_table *ion_map_dma_buf(struct dma_buf_attachment *attachment,
 269                                         enum dma_data_direction direction)
 270 {
 271         struct ion_dma_buf_attachment *a = attachment->priv;
 272         struct sg_table *table;
 273
 274         table = a->table;
 275
 276         if (!dma_map_sg(attachment->dev, table->sgl, table->nents,
 277                         direction))
 278                 return ERR_PTR(-ENOMEM);
 279
 280         return table;
 281 }
 282
 283 static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment,
 284                               struct sg_table *table,
 285                               enum dma_data_direction direction)
 286 {
 287         dma_unmap_sg(attachment->dev, table->sgl, table->nents, direction);
 288 }
 289
 290 static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
 291 {
 292         struct ion_buffer *buffer = dmabuf->priv;
 293         int ret = 0;
 294
 295         if (!buffer->heap->ops->map_user) {
 296                 pr_err("%s: this heap does not define a method for mapping to userspace\n",
 297                        __func__);
 298                 return -EINVAL;
 299         }
 300
 301         if (!(buffer->flags & ION_FLAG_CACHED))
 302                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 303
 304         mutex_lock(&buffer->lock);
 305         /* now map it to userspace */
 306         ret = buffer->heap->ops->map_user(buffer->heap, buffer, vma);
 307         mutex_unlock(&buffer->lock);
 308
 309         if (ret)
 310                 pr_err("%s: failure mapping buffer to userspace\n",
 311                        __func__);
 312
 313         return ret;
 314 }
 315
 316 static void ion_dma_buf_release(struct dma_buf *dmabuf)
 317 {
 318         struct ion_buffer *buffer = dmabuf->priv;
 319
 320         _ion_buffer_destroy(buffer);
 321 }
 322
 323 static void *ion_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset)
 324 {
 325         struct ion_buffer *buffer = dmabuf->priv;
 326
 327         return buffer->vaddr + offset * PAGE_SIZE;
 328 }
 329
 330 static void ion_dma_buf_kunmap(struct dma_buf *dmabuf, unsigned long offset,
 331                                void *ptr)
 332 {
 333 }
 334
 335 static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
 336                                         enum dma_data_direction direction)
 337 {
 338         struct ion_buffer *buffer = dmabuf->priv;
 339         void *vaddr;
 340         struct ion_dma_buf_attachment *a;
 341
 342         /*
 343          * TODO: Move this elsewhere because we don't always need a vaddr
 344          */
 345         if (buffer->heap->ops->map_kernel) {
 346                 mutex_lock(&buffer->lock);
 347                 vaddr = ion_buffer_kmap_get(buffer);
 348                 mutex_unlock(&buffer->lock);
 349         }
 350
 351         mutex_lock(&buffer->lock);
 352         list_for_each_entry(a, &buffer->attachments, list) {
 353                 dma_sync_sg_for_cpu(a->dev, a->table->sgl, a->table->nents,
 354                                     direction);
 355         }
 356         mutex_unlock(&buffer->lock);
 357
 358         return 0;
 359 }
 360
 361 static int ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
 362                                       enum dma_data_direction direction)
 363 {
 364         struct ion_buffer *buffer = dmabuf->priv;
 365         struct ion_dma_buf_attachment *a;
 366
 367         if (buffer->heap->ops->map_kernel) {
 368                 mutex_lock(&buffer->lock);
 369                 ion_buffer_kmap_put(buffer);
 370                 mutex_unlock(&buffer->lock);
 371         }
 372
 373         mutex_lock(&buffer->lock);
 374         list_for_each_entry(a, &buffer->attachments, list) {
 375                 dma_sync_sg_for_device(a->dev, a->table->sgl, a->table->nents,
 376                                        direction);
 377         }
 378         mutex_unlock(&buffer->lock);
 379
 380         return 0;
 381 }
 382
 383 static const struct dma_buf_ops dma_buf_ops = {
 384         .map_dma_buf = ion_map_dma_buf,
 385         .unmap_dma_buf = ion_unmap_dma_buf,
 386         .mmap = ion_mmap,
 387         .release = ion_dma_buf_release,
 388         .attach = ion_dma_buf_attach,
 389         .detach = ion_dma_buf_detatch,
 390         .begin_cpu_access = ion_dma_buf_begin_cpu_access,
 391         .end_cpu_access = ion_dma_buf_end_cpu_access,
 392         .map_atomic = ion_dma_buf_kmap,
 393         .unmap_atomic = ion_dma_buf_kunmap,
 394         .map = ion_dma_buf_kmap,
 395         .unmap = ion_dma_buf_kunmap,
 396 };
 397
 398 int ion_alloc(size_t len, unsigned int heap_id_mask, unsigned int flags)
 399 {
 400         struct ion_device *dev = internal_dev;
 401         struct ion_buffer *buffer = NULL;
 402         struct ion_heap *heap;
 403         DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 404         int fd;
 405         struct dma_buf *dmabuf;
 406
 407         pr_debug("%s: len %zu heap_id_mask %u flags %x\n", __func__,
 408                  len, heap_id_mask, flags);
 409         /*
 410          * traverse the list of heaps available in this system in priority
 411          * order.  If the heap type is supported by the client, and matches the
 412          * request of the caller allocate from it.  Repeat until allocate has
 413          * succeeded or all heaps have been tried
 414          */
 415         len = PAGE_ALIGN(len);
 416
 417         if (!len)
 418                 return -EINVAL;
 419
 420         down_read(&dev->lock);
 421         plist_for_each_entry(heap, &dev->heaps, node) {
 422                 /* if the caller didn't specify this heap id */
 423                 if (!((1 << heap->id) & heap_id_mask))
 424                         continue;
 425                 buffer = ion_buffer_create(heap, dev, len, flags);
 426                 if (!IS_ERR(buffer))
 427                         break;
 428         }
 429         up_read(&dev->lock);
 430
 431         if (!buffer)
 432                 return -ENODEV;
 433
 434         if (IS_ERR(buffer))
 435                 return PTR_ERR(buffer);
 436
 437         exp_info.ops = &dma_buf_ops;
 438         exp_info.size = buffer->size;
 439         exp_info.flags = O_RDWR;
 440         exp_info.priv = buffer;
 441
 442         dmabuf = dma_buf_export(&exp_info);
 443         if (IS_ERR(dmabuf)) {
 444                 _ion_buffer_destroy(buffer);
 445                 return PTR_ERR(dmabuf);
 446         }
 447
 448         fd = dma_buf_fd(dmabuf, O_CLOEXEC);
 449         if (fd < 0)
 450                 dma_buf_put(dmabuf);
 451
 452         return fd;
 453 }
 454
 455 int ion_query_heaps(struct ion_heap_query *query)
 456 {
 457         struct ion_device *dev = internal_dev;
 458         struct ion_heap_data __user *buffer = u64_to_user_ptr(query->heaps);
 459         int ret = -EINVAL, cnt = 0, max_cnt;
 460         struct ion_heap *heap;
 461         struct ion_heap_data hdata;
 462
 463         memset(&hdata, 0, sizeof(hdata));
 464
 465         down_read(&dev->lock);
 466         if (!buffer) {
 467                 query->cnt = dev->heap_cnt;
 468                 ret = 0;
 469                 goto out;
 470         }
 471
 472         if (query->cnt <= 0)
 473                 goto out;
 474
 475         max_cnt = query->cnt;
 476
 477         plist_for_each_entry(heap, &dev->heaps, node) {
 478                 strncpy(hdata.name, heap->name, MAX_HEAP_NAME);
 479                 hdata.name[sizeof(hdata.name) - 1] = '\0';
 480                 hdata.type = heap->type;
 481                 hdata.heap_id = heap->id;
 482
 483                 if (copy_to_user(&buffer[cnt], &hdata, sizeof(hdata))) {
 484                         ret = -EFAULT;
 485                         goto out;
 486                 }
 487
 488                 cnt++;
 489                 if (cnt >= max_cnt)
 490                         break;
 491         }
 492
 493         query->cnt = cnt;
 494         ret = 0;
 495 out:
 496         up_read(&dev->lock);
 497         return ret;
 498 }
 499
 500 static const struct file_operations ion_fops = {
 501         .owner          = THIS_MODULE,
 502         .unlocked_ioctl = ion_ioctl,
 503 #ifdef CONFIG_COMPAT
 504         .compat_ioctl   = ion_ioctl,
 505 #endif
 506 };
 507
 508 static int debug_shrink_set(void *data, u64 val)
 509 {
 510         struct ion_heap *heap = data;
 511         struct shrink_control sc;
 512         int objs;
 513
 514         sc.gfp_mask = GFP_HIGHUSER;
 515         sc.nr_to_scan = val;
 516
 517         if (!val) {
 518                 objs = heap->shrinker.count_objects(&heap->shrinker, &sc);
 519                 sc.nr_to_scan = objs;
 520         }
 521
 522         heap->shrinker.scan_objects(&heap->shrinker, &sc);
 523         return 0;
 524 }
 525
 526 static int debug_shrink_get(void *data, u64 *val)
 527 {
 528         struct ion_heap *heap = data;
 529         struct shrink_control sc;
 530         int objs;
 531
 532         sc.gfp_mask = GFP_HIGHUSER;
 533         sc.nr_to_scan = 0;
 534
 535         objs = heap->shrinker.count_objects(&heap->shrinker, &sc);
 536         *val = objs;
 537         return 0;
 538 }
 539
 540 DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops, debug_shrink_get,
 541                         debug_shrink_set, "%llu\n");
 542
 543 void ion_device_add_heap(struct ion_heap *heap)
 544 {
 545         struct dentry *debug_file;
 546         struct ion_device *dev = internal_dev;
 547
 548         if (!heap->ops->allocate || !heap->ops->free)
 549                 pr_err("%s: can not add heap with invalid ops struct.\n",
 550                        __func__);
 551
 552         spin_lock_init(&heap->free_lock);
 553         heap->free_list_size = 0;
 554
 555         if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
 556                 ion_heap_init_deferred_free(heap);
 557
 558         if ((heap->flags & ION_HEAP_FLAG_DEFER_FREE) || heap->ops->shrink)
 559                 ion_heap_init_shrinker(heap);
 560
 561         heap->dev = dev;
 562         down_write(&dev->lock);
 563         heap->id = heap_id++;
 564         /*
 565          * use negative heap->id to reverse the priority -- when traversing
 566          * the list later attempt higher id numbers first
 567          */
 568         plist_node_init(&heap->node, -heap->id);
 569         plist_add(&heap->node, &dev->heaps);
 570
 571         if (heap->shrinker.count_objects && heap->shrinker.scan_objects) {
 572                 char debug_name[64];
 573
 574                 snprintf(debug_name, 64, "%s_shrink", heap->name);
 575                 debug_file = debugfs_create_file(
 576                         debug_name, 0644, dev->debug_root, heap,
 577                         &debug_shrink_fops);
 578                 if (!debug_file) {
 579                         char buf[256], *path;
 580
 581                         path = dentry_path(dev->debug_root, buf, 256);
 582                         pr_err("Failed to create heap shrinker debugfs at %s/%s\n",
 583                                path, debug_name);
 584                 }
 585         }
 586
 587         dev->heap_cnt++;
 588         up_write(&dev->lock);
 589 }
 590 EXPORT_SYMBOL(ion_device_add_heap);
 591
 592 static int ion_device_create(void)
 593 {
 594         struct ion_device *idev;
 595         int ret;
 596
 597         idev = kzalloc(sizeof(*idev), GFP_KERNEL);
 598         if (!idev)
 599                 return -ENOMEM;
 600
 601         idev->dev.minor = MISC_DYNAMIC_MINOR;
 602         idev->dev.name = "ion";
 603         idev->dev.fops = &ion_fops;
 604         idev->dev.parent = NULL;
 605         ret = misc_register(&idev->dev);
 606         if (ret) {
 607                 pr_err("ion: failed to register misc device.\n");
 608                 kfree(idev);
 609                 return ret;
 610         }
 611
 612         idev->debug_root = debugfs_create_dir("ion", NULL);
 613         if (!idev->debug_root) {
 614                 pr_err("ion: failed to create debugfs root directory.\n");
 615                 goto debugfs_done;
 616         }
 617
 618 debugfs_done:
 619         idev->buffers = RB_ROOT;
 620         mutex_init(&idev->buffer_lock);
 621         init_rwsem(&idev->lock);
 622         plist_head_init(&idev->heaps);
 623         internal_dev = idev;
 624         return 0;
 625 }
 626 subsys_initcall(ion_device_create);