drivers/gpu/drm/panfrost/panfrost_gem.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
   3
   4 #include <linux/err.h>
   5 #include <linux/slab.h>
   6 #include <linux/dma-buf.h>
   7 #include <linux/dma-mapping.h>
   8
   9 #include <drm/panfrost_drm.h>
  10 #include "panfrost_device.h"
  11 #include "panfrost_gem.h"
  12 #include "panfrost_mmu.h"
  13
  14 /* Called DRM core on the last userspace/kernel unreference of the
  15  * BO.
  16  */
  17 static void panfrost_gem_free_object(struct drm_gem_object *obj)
  18 {
  19         struct panfrost_gem_object *bo = to_panfrost_bo(obj);
  20         struct panfrost_device *pfdev = obj->dev->dev_private;
  21
  22         /*
  23          * Make sure the BO is no longer inserted in the shrinker list before
  24          * taking care of the destruction itself. If we don't do that we have a
  25          * race condition between this function and what's done in
  26          * panfrost_gem_shrinker_scan().
  27          */
  28         mutex_lock(&pfdev->shrinker_lock);
  29         list_del_init(&bo->base.madv_list);
  30         mutex_unlock(&pfdev->shrinker_lock);
  31
  32         /*
  33          * If we still have mappings attached to the BO, there's a problem in
  34          * our refcounting.
  35          */
  36         WARN_ON_ONCE(!list_empty(&bo->mappings.list));
  37
  38         if (bo->sgts) {
  39                 int i;
  40                 int n_sgt = bo->base.base.size / SZ_2M;
  41
  42                 for (i = 0; i < n_sgt; i++) {
  43                         if (bo->sgts[i].sgl) {
  44                                 dma_unmap_sgtable(pfdev->dev, &bo->sgts[i],
  45                                                   DMA_BIDIRECTIONAL, 0);
  46                                 sg_free_table(&bo->sgts[i]);
  47                         }
  48                 }
  49                 kvfree(bo->sgts);
  50         }
  51
  52         drm_gem_shmem_free(&bo->base);
  53 }
  54
  55 struct panfrost_gem_mapping *
  56 panfrost_gem_mapping_get(struct panfrost_gem_object *bo,
  57                          struct panfrost_file_priv *priv)
  58 {
  59         struct panfrost_gem_mapping *iter, *mapping = NULL;
  60
  61         mutex_lock(&bo->mappings.lock);
  62         list_for_each_entry(iter, &bo->mappings.list, node) {
  63                 if (iter->mmu == priv->mmu) {
  64                         kref_get(&iter->refcount);
  65                         mapping = iter;
  66                         break;
  67                 }
  68         }
  69         mutex_unlock(&bo->mappings.lock);
  70
  71         return mapping;
  72 }
  73
  74 static void
  75 panfrost_gem_teardown_mapping(struct panfrost_gem_mapping *mapping)
  76 {
  77         if (mapping->active)
  78                 panfrost_mmu_unmap(mapping);
  79
  80         spin_lock(&mapping->mmu->mm_lock);
  81         if (drm_mm_node_allocated(&mapping->mmnode))
  82                 drm_mm_remove_node(&mapping->mmnode);
  83         spin_unlock(&mapping->mmu->mm_lock);
  84 }
  85
  86 static void panfrost_gem_mapping_release(struct kref *kref)
  87 {
  88         struct panfrost_gem_mapping *mapping;
  89
  90         mapping = container_of(kref, struct panfrost_gem_mapping, refcount);
  91
  92         panfrost_gem_teardown_mapping(mapping);
  93         drm_gem_object_put(&mapping->obj->base.base);
  94         panfrost_mmu_ctx_put(mapping->mmu);
  95         kfree(mapping);
  96 }
  97
  98 void panfrost_gem_mapping_put(struct panfrost_gem_mapping *mapping)
  99 {
 100         if (!mapping)
 101                 return;
 102
 103         kref_put(&mapping->refcount, panfrost_gem_mapping_release);
 104 }
 105
 106 void panfrost_gem_teardown_mappings_locked(struct panfrost_gem_object *bo)
 107 {
 108         struct panfrost_gem_mapping *mapping;
 109
 110         list_for_each_entry(mapping, &bo->mappings.list, node)
 111                 panfrost_gem_teardown_mapping(mapping);
 112 }
 113
 114 int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_priv)
 115 {
 116         int ret;
 117         size_t size = obj->size;
 118         u64 align;
 119         struct panfrost_gem_object *bo = to_panfrost_bo(obj);
 120         unsigned long color = bo->noexec ? PANFROST_BO_NOEXEC : 0;
 121         struct panfrost_file_priv *priv = file_priv->driver_priv;
 122         struct panfrost_gem_mapping *mapping;
 123
 124         mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
 125         if (!mapping)
 126                 return -ENOMEM;
 127
 128         INIT_LIST_HEAD(&mapping->node);
 129         kref_init(&mapping->refcount);
 130         drm_gem_object_get(obj);
 131         mapping->obj = bo;
 132
 133         /*
 134          * Executable buffers cannot cross a 16MB boundary as the program
 135          * counter is 24-bits. We assume executable buffers will be less than
 136          * 16MB and aligning executable buffers to their size will avoid
 137          * crossing a 16MB boundary.
 138          */
 139         if (!bo->noexec)
 140                 align = size >> PAGE_SHIFT;
 141         else
 142                 align = size >= SZ_2M ? SZ_2M >> PAGE_SHIFT : 0;
 143
 144         mapping->mmu = panfrost_mmu_ctx_get(priv->mmu);
 145         spin_lock(&mapping->mmu->mm_lock);
 146         ret = drm_mm_insert_node_generic(&mapping->mmu->mm, &mapping->mmnode,
 147                                          size >> PAGE_SHIFT, align, color, 0);
 148         spin_unlock(&mapping->mmu->mm_lock);
 149         if (ret)
 150                 goto err;
 151
 152         if (!bo->is_heap) {
 153                 ret = panfrost_mmu_map(mapping);
 154                 if (ret)
 155                         goto err;
 156         }
 157
 158         mutex_lock(&bo->mappings.lock);
 159         WARN_ON(bo->base.madv != PANFROST_MADV_WILLNEED);
 160         list_add_tail(&mapping->node, &bo->mappings.list);
 161         mutex_unlock(&bo->mappings.lock);
 162
 163 err:
 164         if (ret)
 165                 panfrost_gem_mapping_put(mapping);
 166         return ret;
 167 }
 168
 169 void panfrost_gem_close(struct drm_gem_object *obj, struct drm_file *file_priv)
 170 {
 171         struct panfrost_file_priv *priv = file_priv->driver_priv;
 172         struct panfrost_gem_object *bo = to_panfrost_bo(obj);
 173         struct panfrost_gem_mapping *mapping = NULL, *iter;
 174
 175         mutex_lock(&bo->mappings.lock);
 176         list_for_each_entry(iter, &bo->mappings.list, node) {
 177                 if (iter->mmu == priv->mmu) {
 178                         mapping = iter;
 179                         list_del(&iter->node);
 180                         break;
 181                 }
 182         }
 183         mutex_unlock(&bo->mappings.lock);
 184
 185         panfrost_gem_mapping_put(mapping);
 186 }
 187
 188 static int panfrost_gem_pin(struct drm_gem_object *obj)
 189 {
 190         struct panfrost_gem_object *bo = to_panfrost_bo(obj);
 191
 192         if (bo->is_heap)
 193                 return -EINVAL;
 194
 195         return drm_gem_shmem_pin(&bo->base);
 196 }
 197
 198 static const struct drm_gem_object_funcs panfrost_gem_funcs = {
 199         .free = panfrost_gem_free_object,
 200         .open = panfrost_gem_open,
 201         .close = panfrost_gem_close,
 202         .print_info = drm_gem_shmem_object_print_info,
 203         .pin = panfrost_gem_pin,
 204         .unpin = drm_gem_shmem_object_unpin,
 205         .get_sg_table = drm_gem_shmem_object_get_sg_table,
 206         .vmap = drm_gem_shmem_object_vmap,
 207         .vunmap = drm_gem_shmem_object_vunmap,
 208         .mmap = drm_gem_shmem_object_mmap,
 209         .vm_ops = &drm_gem_shmem_vm_ops,
 210 };
 211
 212 /**
 213  * panfrost_gem_create_object - Implementation of driver->gem_create_object.
 214  * @dev: DRM device
 215  * @size: Size in bytes of the memory the object will reference
 216  *
 217  * This lets the GEM helpers allocate object structs for us, and keep
 218  * our BO stats correct.
 219  */
 220 struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size)
 221 {
 222         struct panfrost_device *pfdev = dev->dev_private;
 223         struct panfrost_gem_object *obj;
 224
 225         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
 226         if (!obj)
 227                 return ERR_PTR(-ENOMEM);
 228
 229         INIT_LIST_HEAD(&obj->mappings.list);
 230         mutex_init(&obj->mappings.lock);
 231         obj->base.base.funcs = &panfrost_gem_funcs;
 232         obj->base.map_wc = !pfdev->coherent;
 233
 234         return &obj->base.base;
 235 }
 236
 237 struct panfrost_gem_object *
 238 panfrost_gem_create_with_handle(struct drm_file *file_priv,
 239                                 struct drm_device *dev, size_t size,
 240                                 u32 flags,
 241                                 uint32_t *handle)
 242 {
 243         int ret;
 244         struct drm_gem_shmem_object *shmem;
 245         struct panfrost_gem_object *bo;
 246
 247         /* Round up heap allocations to 2MB to keep fault handling simple */
 248         if (flags & PANFROST_BO_HEAP)
 249                 size = roundup(size, SZ_2M);
 250
 251         shmem = drm_gem_shmem_create(dev, size);
 252         if (IS_ERR(shmem))
 253                 return ERR_CAST(shmem);
 254
 255         bo = to_panfrost_bo(&shmem->base);
 256         bo->noexec = !!(flags & PANFROST_BO_NOEXEC);
 257         bo->is_heap = !!(flags & PANFROST_BO_HEAP);
 258
 259         /*
 260          * Allocate an id of idr table where the obj is registered
 261          * and handle has the id what user can see.
 262          */
 263         ret = drm_gem_handle_create(file_priv, &shmem->base, handle);
 264         /* drop reference from allocate - handle holds it now. */
 265         drm_gem_object_put(&shmem->base);
 266         if (ret)
 267                 return ERR_PTR(ret);
 268
 269         return bo;
 270 }
 271
 272 struct drm_gem_object *
 273 panfrost_gem_prime_import_sg_table(struct drm_device *dev,
 274                                    struct dma_buf_attachment *attach,
 275                                    struct sg_table *sgt)
 276 {
 277         struct drm_gem_object *obj;
 278         struct panfrost_gem_object *bo;
 279
 280         obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
 281         if (IS_ERR(obj))
 282                 return ERR_CAST(obj);
 283
 284         bo = to_panfrost_bo(obj);
 285         bo->noexec = true;
 286
 287         return obj;
 288 }