drivers/tee/tee_shm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2015-2017, 2019-2021 Linaro Limited
   4  */
   5 #include <linux/anon_inodes.h>
   6 #include <linux/device.h>
   7 #include <linux/idr.h>
   8 #include <linux/mm.h>
   9 #include <linux/sched.h>
  10 #include <linux/slab.h>
  11 #include <linux/tee_drv.h>
  12 #include "tee_private.h"
  13
  14 static void tee_shm_release(struct tee_device *teedev, struct tee_shm *shm)
  15 {
  16         if (shm->flags & TEE_SHM_POOL) {
  17                 struct tee_shm_pool_mgr *poolm;
  18
  19                 if (shm->flags & TEE_SHM_DMA_BUF)
  20                         poolm = teedev->pool->dma_buf_mgr;
  21                 else
  22                         poolm = teedev->pool->private_mgr;
  23
  24                 poolm->ops->free(poolm, shm);
  25         } else if (shm->flags & TEE_SHM_REGISTER) {
  26                 size_t n;
  27                 int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);
  28
  29                 if (rc)
  30                         dev_err(teedev->dev.parent,
  31                                 "unregister shm %p failed: %d", shm, rc);
  32
  33                 for (n = 0; n < shm->num_pages; n++)
  34                         put_page(shm->pages[n]);
  35
  36                 kfree(shm->pages);
  37         }
  38
  39         if (shm->ctx)
  40                 teedev_ctx_put(shm->ctx);
  41
  42         kfree(shm);
  43
  44         tee_device_put(teedev);
  45 }
  46
  47 static struct tee_shm *__tee_shm_alloc(struct tee_context *ctx,
  48                                        struct tee_device *teedev,
  49                                        size_t size, u32 flags)
  50 {
  51         struct tee_shm_pool_mgr *poolm = NULL;
  52         struct tee_shm *shm;
  53         void *ret;
  54         int rc;
  55
  56         if (ctx && ctx->teedev != teedev) {
  57                 dev_err(teedev->dev.parent, "ctx and teedev mismatch\n");
  58                 return ERR_PTR(-EINVAL);
  59         }
  60
  61         if (!(flags & TEE_SHM_MAPPED)) {
  62                 dev_err(teedev->dev.parent,
  63                         "only mapped allocations supported\n");
  64                 return ERR_PTR(-EINVAL);
  65         }
  66
  67         if ((flags & ~(TEE_SHM_MAPPED | TEE_SHM_DMA_BUF | TEE_SHM_PRIV))) {
  68                 dev_err(teedev->dev.parent, "invalid shm flags 0x%x", flags);
  69                 return ERR_PTR(-EINVAL);
  70         }
  71
  72         if (!tee_device_get(teedev))
  73                 return ERR_PTR(-EINVAL);
  74
  75         if (!teedev->pool) {
  76                 /* teedev has been detached from driver */
  77                 ret = ERR_PTR(-EINVAL);
  78                 goto err_dev_put;
  79         }
  80
  81         shm = kzalloc(sizeof(*shm), GFP_KERNEL);
  82         if (!shm) {
  83                 ret = ERR_PTR(-ENOMEM);
  84                 goto err_dev_put;
  85         }
  86
  87         refcount_set(&shm->refcount, 1);
  88         shm->flags = flags | TEE_SHM_POOL;
  89         shm->teedev = teedev;
  90         shm->ctx = ctx;
  91         if (flags & TEE_SHM_DMA_BUF)
  92                 poolm = teedev->pool->dma_buf_mgr;
  93         else
  94                 poolm = teedev->pool->private_mgr;
  95
  96         rc = poolm->ops->alloc(poolm, shm, size);
  97         if (rc) {
  98                 ret = ERR_PTR(rc);
  99                 goto err_kfree;
 100         }
 101
 102         mutex_lock(&teedev->mutex);
 103         shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
 104         mutex_unlock(&teedev->mutex);
 105         if (shm->id < 0) {
 106                 ret = ERR_PTR(shm->id);
 107                 goto err_pool_free;
 108         }
 109
 110         if (ctx) {
 111                 teedev_ctx_get(ctx);
 112                 mutex_lock(&teedev->mutex);
 113                 list_add_tail(&shm->link, &ctx->list_shm);
 114                 mutex_unlock(&teedev->mutex);
 115         }
 116
 117         return shm;
 118 err_pool_free:
 119         poolm->ops->free(poolm, shm);
 120 err_kfree:
 121         kfree(shm);
 122 err_dev_put:
 123         tee_device_put(teedev);
 124         return ret;
 125 }
 126
 127 /**
 128  * tee_shm_alloc() - Allocate shared memory
 129  * @ctx:        Context that allocates the shared memory
 130  * @size:       Requested size of shared memory
 131  * @flags:      Flags setting properties for the requested shared memory.
 132  *
 133  * Memory allocated as global shared memory is automatically freed when the
 134  * TEE file pointer is closed. The @flags field uses the bits defined by
 135  * TEE_SHM_* in <linux/tee_drv.h>. TEE_SHM_MAPPED must currently always be
 136  * set. If TEE_SHM_DMA_BUF global shared memory will be allocated and
 137  * associated with a dma-buf handle, else driver private memory.
 138  */
 139 struct tee_shm *tee_shm_alloc(struct tee_context *ctx, size_t size, u32 flags)
 140 {
 141         return __tee_shm_alloc(ctx, ctx->teedev, size, flags);
 142 }
 143 EXPORT_SYMBOL_GPL(tee_shm_alloc);
 144
 145 struct tee_shm *tee_shm_priv_alloc(struct tee_device *teedev, size_t size)
 146 {
 147         return __tee_shm_alloc(NULL, teedev, size, TEE_SHM_MAPPED);
 148 }
 149 EXPORT_SYMBOL_GPL(tee_shm_priv_alloc);
 150
 151 /**
 152  * tee_shm_alloc_kernel_buf() - Allocate shared memory for kernel buffer
 153  * @ctx:        Context that allocates the shared memory
 154  * @size:       Requested size of shared memory
 155  *
 156  * The returned memory registered in secure world and is suitable to be
 157  * passed as a memory buffer in parameter argument to
 158  * tee_client_invoke_func(). The memory allocated is later freed with a
 159  * call to tee_shm_free().
 160  *
 161  * @returns a pointer to 'struct tee_shm'
 162  */
 163 struct tee_shm *tee_shm_alloc_kernel_buf(struct tee_context *ctx, size_t size)
 164 {
 165         return tee_shm_alloc(ctx, size, TEE_SHM_MAPPED);
 166 }
 167 EXPORT_SYMBOL_GPL(tee_shm_alloc_kernel_buf);
 168
 169 struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr,
 170                                  size_t length, u32 flags)
 171 {
 172         struct tee_device *teedev = ctx->teedev;
 173         const u32 req_flags = TEE_SHM_DMA_BUF | TEE_SHM_USER_MAPPED;
 174         struct tee_shm *shm;
 175         void *ret;
 176         int rc;
 177         int num_pages;
 178         unsigned long start;
 179
 180         if (flags != req_flags)
 181                 return ERR_PTR(-ENOTSUPP);
 182
 183         if (!tee_device_get(teedev))
 184                 return ERR_PTR(-EINVAL);
 185
 186         if (!teedev->desc->ops->shm_register ||
 187             !teedev->desc->ops->shm_unregister) {
 188                 tee_device_put(teedev);
 189                 return ERR_PTR(-ENOTSUPP);
 190         }
 191
 192         teedev_ctx_get(ctx);
 193
 194         shm = kzalloc(sizeof(*shm), GFP_KERNEL);
 195         if (!shm) {
 196                 ret = ERR_PTR(-ENOMEM);
 197                 goto err;
 198         }
 199
 200         refcount_set(&shm->refcount, 1);
 201         shm->flags = flags | TEE_SHM_REGISTER;
 202         shm->teedev = teedev;
 203         shm->ctx = ctx;
 204         shm->id = -1;
 205         addr = untagged_addr(addr);
 206         start = rounddown(addr, PAGE_SIZE);
 207         shm->offset = addr - start;
 208         shm->size = length;
 209         num_pages = (roundup(addr + length, PAGE_SIZE) - start) / PAGE_SIZE;
 210         shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);
 211         if (!shm->pages) {
 212                 ret = ERR_PTR(-ENOMEM);
 213                 goto err;
 214         }
 215
 216         rc = get_user_pages_fast(start, num_pages, FOLL_WRITE, shm->pages);
 217         if (rc > 0)
 218                 shm->num_pages = rc;
 219         if (rc != num_pages) {
 220                 if (rc >= 0)
 221                         rc = -ENOMEM;
 222                 ret = ERR_PTR(rc);
 223                 goto err;
 224         }
 225
 226         mutex_lock(&teedev->mutex);
 227         shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
 228         mutex_unlock(&teedev->mutex);
 229
 230         if (shm->id < 0) {
 231                 ret = ERR_PTR(shm->id);
 232                 goto err;
 233         }
 234
 235         rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,
 236                                              shm->num_pages, start);
 237         if (rc) {
 238                 ret = ERR_PTR(rc);
 239                 goto err;
 240         }
 241
 242         mutex_lock(&teedev->mutex);
 243         list_add_tail(&shm->link, &ctx->list_shm);
 244         mutex_unlock(&teedev->mutex);
 245
 246         return shm;
 247 err:
 248         if (shm) {
 249                 size_t n;
 250
 251                 if (shm->id >= 0) {
 252                         mutex_lock(&teedev->mutex);
 253                         idr_remove(&teedev->idr, shm->id);
 254                         mutex_unlock(&teedev->mutex);
 255                 }
 256                 if (shm->pages) {
 257                         for (n = 0; n < shm->num_pages; n++)
 258                                 put_page(shm->pages[n]);
 259                         kfree(shm->pages);
 260                 }
 261         }
 262         kfree(shm);
 263         teedev_ctx_put(ctx);
 264         tee_device_put(teedev);
 265         return ret;
 266 }
 267 EXPORT_SYMBOL_GPL(tee_shm_register);
 268
 269 static int tee_shm_fop_release(struct inode *inode, struct file *filp)
 270 {
 271         tee_shm_put(filp->private_data);
 272         return 0;
 273 }
 274
 275 static int tee_shm_fop_mmap(struct file *filp, struct vm_area_struct *vma)
 276 {
 277         struct tee_shm *shm = filp->private_data;
 278         size_t size = vma->vm_end - vma->vm_start;
 279
 280         /* Refuse sharing shared memory provided by application */
 281         if (shm->flags & TEE_SHM_USER_MAPPED)
 282                 return -EINVAL;
 283
 284         /* check for overflowing the buffer's size */
 285         if (vma->vm_pgoff + vma_pages(vma) > shm->size >> PAGE_SHIFT)
 286                 return -EINVAL;
 287
 288         return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,
 289                                size, vma->vm_page_prot);
 290 }
 291
 292 static const struct file_operations tee_shm_fops = {
 293         .owner = THIS_MODULE,
 294         .release = tee_shm_fop_release,
 295         .mmap = tee_shm_fop_mmap,
 296 };
 297
 298 /**
 299  * tee_shm_get_fd() - Increase reference count and return file descriptor
 300  * @shm:        Shared memory handle
 301  * @returns user space file descriptor to shared memory
 302  */
 303 int tee_shm_get_fd(struct tee_shm *shm)
 304 {
 305         int fd;
 306
 307         if (!(shm->flags & TEE_SHM_DMA_BUF))
 308                 return -EINVAL;
 309
 310         /* matched by tee_shm_put() in tee_shm_op_release() */
 311         refcount_inc(&shm->refcount);
 312         fd = anon_inode_getfd("tee_shm", &tee_shm_fops, shm, O_RDWR);
 313         if (fd < 0)
 314                 tee_shm_put(shm);
 315         return fd;
 316 }
 317
 318 /**
 319  * tee_shm_free() - Free shared memory
 320  * @shm:        Handle to shared memory to free
 321  */
 322 void tee_shm_free(struct tee_shm *shm)
 323 {
 324         tee_shm_put(shm);
 325 }
 326 EXPORT_SYMBOL_GPL(tee_shm_free);
 327
 328 /**
 329  * tee_shm_va2pa() - Get physical address of a virtual address
 330  * @shm:        Shared memory handle
 331  * @va:         Virtual address to tranlsate
 332  * @pa:         Returned physical address
 333  * @returns 0 on success and < 0 on failure
 334  */
 335 int tee_shm_va2pa(struct tee_shm *shm, void *va, phys_addr_t *pa)
 336 {
 337         if (!(shm->flags & TEE_SHM_MAPPED))
 338                 return -EINVAL;
 339         /* Check that we're in the range of the shm */
 340         if ((char *)va < (char *)shm->kaddr)
 341                 return -EINVAL;
 342         if ((char *)va >= ((char *)shm->kaddr + shm->size))
 343                 return -EINVAL;
 344
 345         return tee_shm_get_pa(
 346                         shm, (unsigned long)va - (unsigned long)shm->kaddr, pa);
 347 }
 348 EXPORT_SYMBOL_GPL(tee_shm_va2pa);
 349
 350 /**
 351  * tee_shm_pa2va() - Get virtual address of a physical address
 352  * @shm:        Shared memory handle
 353  * @pa:         Physical address to tranlsate
 354  * @va:         Returned virtual address
 355  * @returns 0 on success and < 0 on failure
 356  */
 357 int tee_shm_pa2va(struct tee_shm *shm, phys_addr_t pa, void **va)
 358 {
 359         if (!(shm->flags & TEE_SHM_MAPPED))
 360                 return -EINVAL;
 361         /* Check that we're in the range of the shm */
 362         if (pa < shm->paddr)
 363                 return -EINVAL;
 364         if (pa >= (shm->paddr + shm->size))
 365                 return -EINVAL;
 366
 367         if (va) {
 368                 void *v = tee_shm_get_va(shm, pa - shm->paddr);
 369
 370                 if (IS_ERR(v))
 371                         return PTR_ERR(v);
 372                 *va = v;
 373         }
 374         return 0;
 375 }
 376 EXPORT_SYMBOL_GPL(tee_shm_pa2va);
 377
 378 /**
 379  * tee_shm_get_va() - Get virtual address of a shared memory plus an offset
 380  * @shm:        Shared memory handle
 381  * @offs:       Offset from start of this shared memory
 382  * @returns virtual address of the shared memory + offs if offs is within
 383  *      the bounds of this shared memory, else an ERR_PTR
 384  */
 385 void *tee_shm_get_va(struct tee_shm *shm, size_t offs)
 386 {
 387         if (!(shm->flags & TEE_SHM_MAPPED))
 388                 return ERR_PTR(-EINVAL);
 389         if (offs >= shm->size)
 390                 return ERR_PTR(-EINVAL);
 391         return (char *)shm->kaddr + offs;
 392 }
 393 EXPORT_SYMBOL_GPL(tee_shm_get_va);
 394
 395 /**
 396  * tee_shm_get_pa() - Get physical address of a shared memory plus an offset
 397  * @shm:        Shared memory handle
 398  * @offs:       Offset from start of this shared memory
 399  * @pa:         Physical address to return
 400  * @returns 0 if offs is within the bounds of this shared memory, else an
 401  *      error code.
 402  */
 403 int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)
 404 {
 405         if (offs >= shm->size)
 406                 return -EINVAL;
 407         if (pa)
 408                 *pa = shm->paddr + offs;
 409         return 0;
 410 }
 411 EXPORT_SYMBOL_GPL(tee_shm_get_pa);
 412
 413 /**
 414  * tee_shm_get_from_id() - Find shared memory object and increase reference
 415  * count
 416  * @ctx:        Context owning the shared memory
 417  * @id:         Id of shared memory object
 418  * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
 419  */
 420 struct tee_shm *tee_shm_get_from_id(struct tee_context *ctx, int id)
 421 {
 422         struct tee_device *teedev;
 423         struct tee_shm *shm;
 424
 425         if (!ctx)
 426                 return ERR_PTR(-EINVAL);
 427
 428         teedev = ctx->teedev;
 429         mutex_lock(&teedev->mutex);
 430         shm = idr_find(&teedev->idr, id);
 431         /*
 432          * If the tee_shm was found in the IDR it must have a refcount
 433          * larger than 0 due to the guarantee in tee_shm_put() below. So
 434          * it's safe to use refcount_inc().
 435          */
 436         if (!shm || shm->ctx != ctx)
 437                 shm = ERR_PTR(-EINVAL);
 438         else
 439                 refcount_inc(&shm->refcount);
 440         mutex_unlock(&teedev->mutex);
 441         return shm;
 442 }
 443 EXPORT_SYMBOL_GPL(tee_shm_get_from_id);
 444
 445 /**
 446  * tee_shm_put() - Decrease reference count on a shared memory handle
 447  * @shm:        Shared memory handle
 448  */
 449 void tee_shm_put(struct tee_shm *shm)
 450 {
 451         struct tee_device *teedev = shm->teedev;
 452         bool do_release = false;
 453
 454         mutex_lock(&teedev->mutex);
 455         if (refcount_dec_and_test(&shm->refcount)) {
 456                 /*
 457                  * refcount has reached 0, we must now remove it from the
 458                  * IDR before releasing the mutex. This will guarantee that
 459                  * the refcount_inc() in tee_shm_get_from_id() never starts
 460                  * from 0.
 461                  */
 462                 idr_remove(&teedev->idr, shm->id);
 463                 if (shm->ctx)
 464                         list_del(&shm->link);
 465                 do_release = true;
 466         }
 467         mutex_unlock(&teedev->mutex);
 468
 469         if (do_release)
 470                 tee_shm_release(teedev, shm);
 471 }
 472 EXPORT_SYMBOL_GPL(tee_shm_put);