drivers/gpu/drm/nouveau/nouveau_gem.c

   1 /*
   2  * Copyright (C) 2008 Ben Skeggs.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sublicense, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial
  15  * portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  */
  26
  27 #include "nouveau_drv.h"
  28 #include "nouveau_dma.h"
  29 #include "nouveau_fence.h"
  30 #include "nouveau_abi16.h"
  31
  32 #include "nouveau_ttm.h"
  33 #include "nouveau_gem.h"
  34 #include "nouveau_mem.h"
  35 #include "nouveau_vmm.h"
  36
  37 #include <nvif/class.h>
  38
  39 void
  40 nouveau_gem_object_del(struct drm_gem_object *gem)
  41 {
  42         struct nouveau_bo *nvbo = nouveau_gem_object(gem);
  43         struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  44         struct ttm_buffer_object *bo = &nvbo->bo;
  45         struct device *dev = drm->dev->dev;
  46         int ret;
  47
  48         ret = pm_runtime_get_sync(dev);
  49         if (WARN_ON(ret < 0 && ret != -EACCES)) {
  50                 pm_runtime_put_autosuspend(dev);
  51                 return;
  52         }
  53
  54         if (gem->import_attach)
  55                 drm_prime_gem_destroy(gem, nvbo->bo.sg);
  56
  57         drm_gem_object_release(gem);
  58
  59         /* reset filp so nouveau_bo_del_ttm() can test for it */
  60         gem->filp = NULL;
  61         ttm_bo_unref(&bo);
  62
  63         pm_runtime_mark_last_busy(dev);
  64         pm_runtime_put_autosuspend(dev);
  65 }
  66
  67 int
  68 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
  69 {
  70         struct nouveau_cli *cli = nouveau_cli(file_priv);
  71         struct nouveau_bo *nvbo = nouveau_gem_object(gem);
  72         struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  73         struct device *dev = drm->dev->dev;
  74         struct nouveau_vma *vma;
  75         int ret;
  76
  77         if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50)
  78                 return 0;
  79
  80         ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
  81         if (ret)
  82                 return ret;
  83
  84         ret = pm_runtime_get_sync(dev);
  85         if (ret < 0 && ret != -EACCES) {
  86                 pm_runtime_put_autosuspend(dev);
  87                 goto out;
  88         }
  89
  90         ret = nouveau_vma_new(nvbo, &cli->vmm, &vma);
  91         pm_runtime_mark_last_busy(dev);
  92         pm_runtime_put_autosuspend(dev);
  93 out:
  94         ttm_bo_unreserve(&nvbo->bo);
  95         return ret;
  96 }
  97
  98 struct nouveau_gem_object_unmap {
  99         struct nouveau_cli_work work;
 100         struct nouveau_vma *vma;
 101 };
 102
 103 static void
 104 nouveau_gem_object_delete(struct nouveau_vma *vma)
 105 {
 106         nouveau_fence_unref(&vma->fence);
 107         nouveau_vma_del(&vma);
 108 }
 109
 110 static void
 111 nouveau_gem_object_delete_work(struct nouveau_cli_work *w)
 112 {
 113         struct nouveau_gem_object_unmap *work =
 114                 container_of(w, typeof(*work), work);
 115         nouveau_gem_object_delete(work->vma);
 116         kfree(work);
 117 }
 118
 119 static void
 120 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
 121 {
 122         struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL;
 123         struct nouveau_gem_object_unmap *work;
 124
 125         list_del_init(&vma->head);
 126
 127         if (!fence) {
 128                 nouveau_gem_object_delete(vma);
 129                 return;
 130         }
 131
 132         if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) {
 133                 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0);
 134                 nouveau_gem_object_delete(vma);
 135                 return;
 136         }
 137
 138         work->work.func = nouveau_gem_object_delete_work;
 139         work->vma = vma;
 140         nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work);
 141 }
 142
 143 void
 144 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
 145 {
 146         struct nouveau_cli *cli = nouveau_cli(file_priv);
 147         struct nouveau_bo *nvbo = nouveau_gem_object(gem);
 148         struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 149         struct device *dev = drm->dev->dev;
 150         struct nouveau_vma *vma;
 151         int ret;
 152
 153         if (cli->vmm.vmm.object.oclass < NVIF_CLASS_VMM_NV50)
 154                 return;
 155
 156         ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
 157         if (ret)
 158                 return;
 159
 160         vma = nouveau_vma_find(nvbo, &cli->vmm);
 161         if (vma) {
 162                 if (--vma->refs == 0) {
 163                         ret = pm_runtime_get_sync(dev);
 164                         if (!WARN_ON(ret < 0 && ret != -EACCES)) {
 165                                 nouveau_gem_object_unmap(nvbo, vma);
 166                                 pm_runtime_mark_last_busy(dev);
 167                                 pm_runtime_put_autosuspend(dev);
 168                         }
 169                 }
 170         }
 171         ttm_bo_unreserve(&nvbo->bo);
 172 }
 173
 174 int
 175 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
 176                 uint32_t tile_mode, uint32_t tile_flags,
 177                 struct nouveau_bo **pnvbo)
 178 {
 179         struct nouveau_drm *drm = cli->drm;
 180         struct nouveau_bo *nvbo;
 181         u32 flags = 0;
 182         int ret;
 183
 184         if (domain & NOUVEAU_GEM_DOMAIN_VRAM)
 185                 flags |= TTM_PL_FLAG_VRAM;
 186         if (domain & NOUVEAU_GEM_DOMAIN_GART)
 187                 flags |= TTM_PL_FLAG_TT;
 188         if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU)
 189                 flags |= TTM_PL_FLAG_SYSTEM;
 190
 191         if (domain & NOUVEAU_GEM_DOMAIN_COHERENT)
 192                 flags |= TTM_PL_FLAG_UNCACHED;
 193
 194         ret = nouveau_bo_new(cli, size, align, flags, tile_mode,
 195                              tile_flags, NULL, NULL, pnvbo);
 196         if (ret)
 197                 return ret;
 198         nvbo = *pnvbo;
 199
 200         /* we restrict allowed domains on nv50+ to only the types
 201          * that were requested at creation time.  not possibly on
 202          * earlier chips without busting the ABI.
 203          */
 204         nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
 205                               NOUVEAU_GEM_DOMAIN_GART;
 206         if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
 207                 nvbo->valid_domains &= domain;
 208
 209         /* Initialize the embedded gem-object. We return a single gem-reference
 210          * to the caller, instead of a normal nouveau_bo ttm reference. */
 211         ret = drm_gem_object_init(drm->dev, &nvbo->gem, nvbo->bo.mem.size);
 212         if (ret) {
 213                 nouveau_bo_ref(NULL, pnvbo);
 214                 return -ENOMEM;
 215         }
 216
 217         nvbo->bo.persistent_swap_storage = nvbo->gem.filp;
 218         return 0;
 219 }
 220
 221 static int
 222 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
 223                  struct drm_nouveau_gem_info *rep)
 224 {
 225         struct nouveau_cli *cli = nouveau_cli(file_priv);
 226         struct nouveau_bo *nvbo = nouveau_gem_object(gem);
 227         struct nouveau_vma *vma;
 228
 229         if (is_power_of_2(nvbo->valid_domains))
 230                 rep->domain = nvbo->valid_domains;
 231         else if (nvbo->bo.mem.mem_type == TTM_PL_TT)
 232                 rep->domain = NOUVEAU_GEM_DOMAIN_GART;
 233         else
 234                 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
 235         rep->offset = nvbo->bo.offset;
 236         if (cli->vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
 237                 vma = nouveau_vma_find(nvbo, &cli->vmm);
 238                 if (!vma)
 239                         return -EINVAL;
 240
 241                 rep->offset = vma->addr;
 242         }
 243
 244         rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
 245         rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node);
 246         rep->tile_mode = nvbo->mode;
 247         rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG;
 248         if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI)
 249                 rep->tile_flags |= nvbo->kind << 8;
 250         else
 251         if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
 252                 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16;
 253         else
 254                 rep->tile_flags |= nvbo->zeta;
 255         return 0;
 256 }
 257
 258 int
 259 nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
 260                       struct drm_file *file_priv)
 261 {
 262         struct nouveau_cli *cli = nouveau_cli(file_priv);
 263         struct drm_nouveau_gem_new *req = data;
 264         struct nouveau_bo *nvbo = NULL;
 265         int ret = 0;
 266
 267         ret = nouveau_gem_new(cli, req->info.size, req->align,
 268                               req->info.domain, req->info.tile_mode,
 269                               req->info.tile_flags, &nvbo);
 270         if (ret)
 271                 return ret;
 272
 273         ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle);
 274         if (ret == 0) {
 275                 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info);
 276                 if (ret)
 277                         drm_gem_handle_delete(file_priv, req->info.handle);
 278         }
 279
 280         /* drop reference from allocate - handle holds it now */
 281         drm_gem_object_put_unlocked(&nvbo->gem);
 282         return ret;
 283 }
 284
 285 static int
 286 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
 287                        uint32_t write_domains, uint32_t valid_domains)
 288 {
 289         struct nouveau_bo *nvbo = nouveau_gem_object(gem);
 290         struct ttm_buffer_object *bo = &nvbo->bo;
 291         uint32_t domains = valid_domains & nvbo->valid_domains &
 292                 (write_domains ? write_domains : read_domains);
 293         uint32_t pref_flags = 0, valid_flags = 0;
 294
 295         if (!domains)
 296                 return -EINVAL;
 297
 298         if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
 299                 valid_flags |= TTM_PL_FLAG_VRAM;
 300
 301         if (valid_domains & NOUVEAU_GEM_DOMAIN_GART)
 302                 valid_flags |= TTM_PL_FLAG_TT;
 303
 304         if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
 305             bo->mem.mem_type == TTM_PL_VRAM)
 306                 pref_flags |= TTM_PL_FLAG_VRAM;
 307
 308         else if ((domains & NOUVEAU_GEM_DOMAIN_GART) &&
 309                  bo->mem.mem_type == TTM_PL_TT)
 310                 pref_flags |= TTM_PL_FLAG_TT;
 311
 312         else if (domains & NOUVEAU_GEM_DOMAIN_VRAM)
 313                 pref_flags |= TTM_PL_FLAG_VRAM;
 314
 315         else
 316                 pref_flags |= TTM_PL_FLAG_TT;
 317
 318         nouveau_bo_placement_set(nvbo, pref_flags, valid_flags);
 319
 320         return 0;
 321 }
 322
 323 struct validate_op {
 324         struct list_head list;
 325         struct ww_acquire_ctx ticket;
 326 };
 327
 328 static void
 329 validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence,
 330                         struct drm_nouveau_gem_pushbuf_bo *pbbo)
 331 {
 332         struct nouveau_bo *nvbo;
 333         struct drm_nouveau_gem_pushbuf_bo *b;
 334
 335         while (!list_empty(&op->list)) {
 336                 nvbo = list_entry(op->list.next, struct nouveau_bo, entry);
 337                 b = &pbbo[nvbo->pbbo_index];
 338
 339                 if (likely(fence)) {
 340                         struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 341                         struct nouveau_vma *vma;
 342
 343                         nouveau_bo_fence(nvbo, fence, !!b->write_domains);
 344
 345                         if (drm->client.vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
 346                                 vma = (void *)(unsigned long)b->user_priv;
 347                                 nouveau_fence_unref(&vma->fence);
 348                                 dma_fence_get(&fence->base);
 349                                 vma->fence = fence;
 350                         }
 351                 }
 352
 353                 if (unlikely(nvbo->validate_mapped)) {
 354                         ttm_bo_kunmap(&nvbo->kmap);
 355                         nvbo->validate_mapped = false;
 356                 }
 357
 358                 list_del(&nvbo->entry);
 359                 nvbo->reserved_by = NULL;
 360                 ttm_bo_unreserve(&nvbo->bo);
 361                 drm_gem_object_put_unlocked(&nvbo->gem);
 362         }
 363 }
 364
 365 static void
 366 validate_fini(struct validate_op *op, struct nouveau_fence *fence,
 367               struct drm_nouveau_gem_pushbuf_bo *pbbo)
 368 {
 369         validate_fini_no_ticket(op, fence, pbbo);
 370         ww_acquire_fini(&op->ticket);
 371 }
 372
 373 static int
 374 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
 375               struct drm_nouveau_gem_pushbuf_bo *pbbo,
 376               int nr_buffers, struct validate_op *op)
 377 {
 378         struct nouveau_cli *cli = nouveau_cli(file_priv);
 379         int trycnt = 0;
 380         int ret = -EINVAL, i;
 381         struct nouveau_bo *res_bo = NULL;
 382         LIST_HEAD(gart_list);
 383         LIST_HEAD(vram_list);
 384         LIST_HEAD(both_list);
 385
 386         ww_acquire_init(&op->ticket, &reservation_ww_class);
 387 retry:
 388         if (++trycnt > 100000) {
 389                 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__);
 390                 return -EINVAL;
 391         }
 392
 393         for (i = 0; i < nr_buffers; i++) {
 394                 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i];
 395                 struct drm_gem_object *gem;
 396                 struct nouveau_bo *nvbo;
 397
 398                 gem = drm_gem_object_lookup(file_priv, b->handle);
 399                 if (!gem) {
 400                         NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle);
 401                         ret = -ENOENT;
 402                         break;
 403                 }
 404                 nvbo = nouveau_gem_object(gem);
 405                 if (nvbo == res_bo) {
 406                         res_bo = NULL;
 407                         drm_gem_object_put_unlocked(gem);
 408                         continue;
 409                 }
 410
 411                 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
 412                         NV_PRINTK(err, cli, "multiple instances of buffer %d on "
 413                                       "validation list\n", b->handle);
 414                         drm_gem_object_put_unlocked(gem);
 415                         ret = -EINVAL;
 416                         break;
 417                 }
 418
 419                 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket);
 420                 if (ret) {
 421                         list_splice_tail_init(&vram_list, &op->list);
 422                         list_splice_tail_init(&gart_list, &op->list);
 423                         list_splice_tail_init(&both_list, &op->list);
 424                         validate_fini_no_ticket(op, NULL, NULL);
 425                         if (unlikely(ret == -EDEADLK)) {
 426                                 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,
 427                                                               &op->ticket);
 428                                 if (!ret)
 429                                         res_bo = nvbo;
 430                         }
 431                         if (unlikely(ret)) {
 432                                 if (ret != -ERESTARTSYS)
 433                                         NV_PRINTK(err, cli, "fail reserve\n");
 434                                 break;
 435                         }
 436                 }
 437
 438                 if (cli->vmm.vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
 439                         struct nouveau_vmm *vmm = &cli->vmm;
 440                         struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm);
 441                         if (!vma) {
 442                                 NV_PRINTK(err, cli, "vma not found!\n");
 443                                 ret = -EINVAL;
 444                                 break;
 445                         }
 446
 447                         b->user_priv = (uint64_t)(unsigned long)vma;
 448                 } else {
 449                         b->user_priv = (uint64_t)(unsigned long)nvbo;
 450                 }
 451
 452                 nvbo->reserved_by = file_priv;
 453                 nvbo->pbbo_index = i;
 454                 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
 455                     (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
 456                         list_add_tail(&nvbo->entry, &both_list);
 457                 else
 458                 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
 459                         list_add_tail(&nvbo->entry, &vram_list);
 460                 else
 461                 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
 462                         list_add_tail(&nvbo->entry, &gart_list);
 463                 else {
 464                         NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n",
 465                                  b->valid_domains);
 466                         list_add_tail(&nvbo->entry, &both_list);
 467                         ret = -EINVAL;
 468                         break;
 469                 }
 470                 if (nvbo == res_bo)
 471                         goto retry;
 472         }
 473
 474         ww_acquire_done(&op->ticket);
 475         list_splice_tail(&vram_list, &op->list);
 476         list_splice_tail(&gart_list, &op->list);
 477         list_splice_tail(&both_list, &op->list);
 478         if (ret)
 479                 validate_fini(op, NULL, NULL);
 480         return ret;
 481
 482 }
 483
 484 static int
 485 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli,
 486               struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo,
 487               uint64_t user_pbbo_ptr)
 488 {
 489         struct nouveau_drm *drm = chan->drm;
 490         struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
 491                                 (void __force __user *)(uintptr_t)user_pbbo_ptr;
 492         struct nouveau_bo *nvbo;
 493         int ret, relocs = 0;
 494
 495         list_for_each_entry(nvbo, list, entry) {
 496                 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];
 497
 498                 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains,
 499                                              b->write_domains,
 500                                              b->valid_domains);
 501                 if (unlikely(ret)) {
 502                         NV_PRINTK(err, cli, "fail set_domain\n");
 503                         return ret;
 504                 }
 505
 506                 ret = nouveau_bo_validate(nvbo, true, false);
 507                 if (unlikely(ret)) {
 508                         if (ret != -ERESTARTSYS)
 509                                 NV_PRINTK(err, cli, "fail ttm_validate\n");
 510                         return ret;
 511                 }
 512
 513                 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true);
 514                 if (unlikely(ret)) {
 515                         if (ret != -ERESTARTSYS)
 516                                 NV_PRINTK(err, cli, "fail post-validate sync\n");
 517                         return ret;
 518                 }
 519
 520                 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
 521                         if (nvbo->bo.offset == b->presumed.offset &&
 522                             ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
 523                               b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
 524                              (nvbo->bo.mem.mem_type == TTM_PL_TT &&
 525                               b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
 526                                 continue;
 527
 528                         if (nvbo->bo.mem.mem_type == TTM_PL_TT)
 529                                 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
 530                         else
 531                                 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
 532                         b->presumed.offset = nvbo->bo.offset;
 533                         b->presumed.valid = 0;
 534                         relocs++;
 535
 536                         if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed,
 537                                              &b->presumed, sizeof(b->presumed)))
 538                                 return -EFAULT;
 539                 }
 540         }
 541
 542         return relocs;
 543 }
 544
 545 static int
 546 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
 547                              struct drm_file *file_priv,
 548                              struct drm_nouveau_gem_pushbuf_bo *pbbo,
 549                              uint64_t user_buffers, int nr_buffers,
 550                              struct validate_op *op, int *apply_relocs)
 551 {
 552         struct nouveau_cli *cli = nouveau_cli(file_priv);
 553         int ret;
 554
 555         INIT_LIST_HEAD(&op->list);
 556
 557         if (nr_buffers == 0)
 558                 return 0;
 559
 560         ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
 561         if (unlikely(ret)) {
 562                 if (ret != -ERESTARTSYS)
 563                         NV_PRINTK(err, cli, "validate_init\n");
 564                 return ret;
 565         }
 566
 567         ret = validate_list(chan, cli, &op->list, pbbo, user_buffers);
 568         if (unlikely(ret < 0)) {
 569                 if (ret != -ERESTARTSYS)
 570                         NV_PRINTK(err, cli, "validating bo list\n");
 571                 validate_fini(op, NULL, NULL);
 572                 return ret;
 573         }
 574         *apply_relocs = ret;
 575         return 0;
 576 }
 577
 578 static inline void
 579 u_free(void *addr)
 580 {
 581         kvfree(addr);
 582 }
 583
 584 static inline void *
 585 u_memcpya(uint64_t user, unsigned nmemb, unsigned size)
 586 {
 587         void *mem;
 588         void __user *userptr = (void __force __user *)(uintptr_t)user;
 589
 590         size *= nmemb;
 591
 592         mem = kvmalloc(size, GFP_KERNEL);
 593         if (!mem)
 594                 return ERR_PTR(-ENOMEM);
 595
 596         if (copy_from_user(mem, userptr, size)) {
 597                 u_free(mem);
 598                 return ERR_PTR(-EFAULT);
 599         }
 600
 601         return mem;
 602 }
 603
 604 static int
 605 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
 606                                 struct drm_nouveau_gem_pushbuf *req,
 607                                 struct drm_nouveau_gem_pushbuf_bo *bo)
 608 {
 609         struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
 610         int ret = 0;
 611         unsigned i;
 612
 613         reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc));
 614         if (IS_ERR(reloc))
 615                 return PTR_ERR(reloc);
 616
 617         for (i = 0; i < req->nr_relocs; i++) {
 618                 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i];
 619                 struct drm_nouveau_gem_pushbuf_bo *b;
 620                 struct nouveau_bo *nvbo;
 621                 uint32_t data;
 622
 623                 if (unlikely(r->bo_index >= req->nr_buffers)) {
 624                         NV_PRINTK(err, cli, "reloc bo index invalid\n");
 625                         ret = -EINVAL;
 626                         break;
 627                 }
 628
 629                 b = &bo[r->bo_index];
 630                 if (b->presumed.valid)
 631                         continue;
 632
 633                 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) {
 634                         NV_PRINTK(err, cli, "reloc container bo index invalid\n");
 635                         ret = -EINVAL;
 636                         break;
 637                 }
 638                 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv;
 639
 640                 if (unlikely(r->reloc_bo_offset + 4 >
 641                              nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
 642                         NV_PRINTK(err, cli, "reloc outside of bo\n");
 643                         ret = -EINVAL;
 644                         break;
 645                 }
 646
 647                 if (!nvbo->kmap.virtual) {
 648                         ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
 649                                           &nvbo->kmap);
 650                         if (ret) {
 651                                 NV_PRINTK(err, cli, "failed kmap for reloc\n");
 652                                 break;
 653                         }
 654                         nvbo->validate_mapped = true;
 655                 }
 656
 657                 if (r->flags & NOUVEAU_GEM_RELOC_LOW)
 658                         data = b->presumed.offset + r->data;
 659                 else
 660                 if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
 661                         data = (b->presumed.offset + r->data) >> 32;
 662                 else
 663                         data = r->data;
 664
 665                 if (r->flags & NOUVEAU_GEM_RELOC_OR) {
 666                         if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART)
 667                                 data |= r->tor;
 668                         else
 669                                 data |= r->vor;
 670                 }
 671
 672                 ret = ttm_bo_wait(&nvbo->bo, false, false);
 673                 if (ret) {
 674                         NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret);
 675                         break;
 676                 }
 677
 678                 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data);
 679         }
 680
 681         u_free(reloc);
 682         return ret;
 683 }
 684
 685 int
 686 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
 687                           struct drm_file *file_priv)
 688 {
 689         struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
 690         struct nouveau_cli *cli = nouveau_cli(file_priv);
 691         struct nouveau_abi16_chan *temp;
 692         struct nouveau_drm *drm = nouveau_drm(dev);
 693         struct drm_nouveau_gem_pushbuf *req = data;
 694         struct drm_nouveau_gem_pushbuf_push *push;
 695         struct drm_nouveau_gem_pushbuf_bo *bo;
 696         struct nouveau_channel *chan = NULL;
 697         struct validate_op op;
 698         struct nouveau_fence *fence = NULL;
 699         int i, j, ret = 0, do_reloc = 0;
 700
 701         if (unlikely(!abi16))
 702                 return -ENOMEM;
 703
 704         list_for_each_entry(temp, &abi16->channels, head) {
 705                 if (temp->chan->chid == req->channel) {
 706                         chan = temp->chan;
 707                         break;
 708                 }
 709         }
 710
 711         if (!chan)
 712                 return nouveau_abi16_put(abi16, -ENOENT);
 713
 714         req->vram_available = drm->gem.vram_available;
 715         req->gart_available = drm->gem.gart_available;
 716         if (unlikely(req->nr_push == 0))
 717                 goto out_next;
 718
 719         if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
 720                 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n",
 721                          req->nr_push, NOUVEAU_GEM_MAX_PUSH);
 722                 return nouveau_abi16_put(abi16, -EINVAL);
 723         }
 724
 725         if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
 726                 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n",
 727                          req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
 728                 return nouveau_abi16_put(abi16, -EINVAL);
 729         }
 730
 731         if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
 732                 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n",
 733                          req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
 734                 return nouveau_abi16_put(abi16, -EINVAL);
 735         }
 736
 737         push = u_memcpya(req->push, req->nr_push, sizeof(*push));
 738         if (IS_ERR(push))
 739                 return nouveau_abi16_put(abi16, PTR_ERR(push));
 740
 741         bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
 742         if (IS_ERR(bo)) {
 743                 u_free(push);
 744                 return nouveau_abi16_put(abi16, PTR_ERR(bo));
 745         }
 746
 747         /* Ensure all push buffers are on validate list */
 748         for (i = 0; i < req->nr_push; i++) {
 749                 if (push[i].bo_index >= req->nr_buffers) {
 750                         NV_PRINTK(err, cli, "push %d buffer not in list\n", i);
 751                         ret = -EINVAL;
 752                         goto out_prevalid;
 753                 }
 754         }
 755
 756         /* Validate buffer list */
 757         ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers,
 758                                            req->nr_buffers, &op, &do_reloc);
 759         if (ret) {
 760                 if (ret != -ERESTARTSYS)
 761                         NV_PRINTK(err, cli, "validate: %d\n", ret);
 762                 goto out_prevalid;
 763         }
 764
 765         /* Apply any relocations that are required */
 766         if (do_reloc) {
 767                 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo);
 768                 if (ret) {
 769                         NV_PRINTK(err, cli, "reloc apply: %d\n", ret);
 770                         goto out;
 771                 }
 772         }
 773
 774         if (chan->dma.ib_max) {
 775                 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16);
 776                 if (ret) {
 777                         NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret);
 778                         goto out;
 779                 }
 780
 781                 for (i = 0; i < req->nr_push; i++) {
 782                         struct nouveau_vma *vma = (void *)(unsigned long)
 783                                 bo[push[i].bo_index].user_priv;
 784
 785                         nv50_dma_push(chan, vma->addr + push[i].offset,
 786                                       push[i].length);
 787                 }
 788         } else
 789         if (drm->client.device.info.chipset >= 0x25) {
 790                 ret = RING_SPACE(chan, req->nr_push * 2);
 791                 if (ret) {
 792                         NV_PRINTK(err, cli, "cal_space: %d\n", ret);
 793                         goto out;
 794                 }
 795
 796                 for (i = 0; i < req->nr_push; i++) {
 797                         struct nouveau_bo *nvbo = (void *)(unsigned long)
 798                                 bo[push[i].bo_index].user_priv;
 799
 800                         OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2);
 801                         OUT_RING(chan, 0);
 802                 }
 803         } else {
 804                 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
 805                 if (ret) {
 806                         NV_PRINTK(err, cli, "jmp_space: %d\n", ret);
 807                         goto out;
 808                 }
 809
 810                 for (i = 0; i < req->nr_push; i++) {
 811                         struct nouveau_bo *nvbo = (void *)(unsigned long)
 812                                 bo[push[i].bo_index].user_priv;
 813                         uint32_t cmd;
 814
 815                         cmd = chan->push.addr + ((chan->dma.cur + 2) << 2);
 816                         cmd |= 0x20000000;
 817                         if (unlikely(cmd != req->suffix0)) {
 818                                 if (!nvbo->kmap.virtual) {
 819                                         ret = ttm_bo_kmap(&nvbo->bo, 0,
 820                                                           nvbo->bo.mem.
 821                                                           num_pages,
 822                                                           &nvbo->kmap);
 823                                         if (ret) {
 824                                                 WIND_RING(chan);
 825                                                 goto out;
 826                                         }
 827                                         nvbo->validate_mapped = true;
 828                                 }
 829
 830                                 nouveau_bo_wr32(nvbo, (push[i].offset +
 831                                                 push[i].length - 8) / 4, cmd);
 832                         }
 833
 834                         OUT_RING(chan, 0x20000000 |
 835                                       (nvbo->bo.offset + push[i].offset));
 836                         OUT_RING(chan, 0);
 837                         for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
 838                                 OUT_RING(chan, 0);
 839                 }
 840         }
 841
 842         ret = nouveau_fence_new(chan, false, &fence);
 843         if (ret) {
 844                 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret);
 845                 WIND_RING(chan);
 846                 goto out;
 847         }
 848
 849 out:
 850         validate_fini(&op, fence, bo);
 851         nouveau_fence_unref(&fence);
 852
 853 out_prevalid:
 854         u_free(bo);
 855         u_free(push);
 856
 857 out_next:
 858         if (chan->dma.ib_max) {
 859                 req->suffix0 = 0x00000000;
 860                 req->suffix1 = 0x00000000;
 861         } else
 862         if (drm->client.device.info.chipset >= 0x25) {
 863                 req->suffix0 = 0x00020000;
 864                 req->suffix1 = 0x00000000;
 865         } else {
 866                 req->suffix0 = 0x20000000 |
 867                               (chan->push.addr + ((chan->dma.cur + 2) << 2));
 868                 req->suffix1 = 0x00000000;
 869         }
 870
 871         return nouveau_abi16_put(abi16, ret);
 872 }
 873
 874 int
 875 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
 876                            struct drm_file *file_priv)
 877 {
 878         struct drm_nouveau_gem_cpu_prep *req = data;
 879         struct drm_gem_object *gem;
 880         struct nouveau_bo *nvbo;
 881         bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
 882         bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE);
 883         long lret;
 884         int ret;
 885
 886         gem = drm_gem_object_lookup(file_priv, req->handle);
 887         if (!gem)
 888                 return -ENOENT;
 889         nvbo = nouveau_gem_object(gem);
 890
 891         lret = reservation_object_wait_timeout_rcu(nvbo->bo.resv, write, true,
 892                                                    no_wait ? 0 : 30 * HZ);
 893         if (!lret)
 894                 ret = -EBUSY;
 895         else if (lret > 0)
 896                 ret = 0;
 897         else
 898                 ret = lret;
 899
 900         nouveau_bo_sync_for_cpu(nvbo);
 901         drm_gem_object_put_unlocked(gem);
 902
 903         return ret;
 904 }
 905
 906 int
 907 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
 908                            struct drm_file *file_priv)
 909 {
 910         struct drm_nouveau_gem_cpu_fini *req = data;
 911         struct drm_gem_object *gem;
 912         struct nouveau_bo *nvbo;
 913
 914         gem = drm_gem_object_lookup(file_priv, req->handle);
 915         if (!gem)
 916                 return -ENOENT;
 917         nvbo = nouveau_gem_object(gem);
 918
 919         nouveau_bo_sync_for_device(nvbo);
 920         drm_gem_object_put_unlocked(gem);
 921         return 0;
 922 }
 923
 924 int
 925 nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
 926                        struct drm_file *file_priv)
 927 {
 928         struct drm_nouveau_gem_info *req = data;
 929         struct drm_gem_object *gem;
 930         int ret;
 931
 932         gem = drm_gem_object_lookup(file_priv, req->handle);
 933         if (!gem)
 934                 return -ENOENT;
 935
 936         ret = nouveau_gem_info(file_priv, gem, req);
 937         drm_gem_object_put_unlocked(gem);
 938         return ret;
 939 }
 940