drivers/gpu/host1x/cdma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Tegra host1x Command DMA
   4  *
   5  * Copyright (c) 2010-2013, NVIDIA Corporation.
   6  */
   7
   8
   9 #include <asm/cacheflush.h>
  10 #include <linux/device.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/host1x.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/kernel.h>
  15 #include <linux/kfifo.h>
  16 #include <linux/slab.h>
  17 #include <trace/events/host1x.h>
  18
  19 #include "cdma.h"
  20 #include "channel.h"
  21 #include "dev.h"
  22 #include "debug.h"
  23 #include "job.h"
  24
  25 /*
  26  * push_buffer
  27  *
  28  * The push buffer is a circular array of words to be fetched by command DMA.
  29  * Note that it works slightly differently to the sync queue; fence == pos
  30  * means that the push buffer is full, not empty.
  31  */
  32
  33 /*
  34  * Typically the commands written into the push buffer are a pair of words. We
  35  * use slots to represent each of these pairs and to simplify things. Note the
  36  * strange number of slots allocated here. 512 slots will fit exactly within a
  37  * single memory page. We also need one additional word at the end of the push
  38  * buffer for the RESTART opcode that will instruct the CDMA to jump back to
  39  * the beginning of the push buffer. With 512 slots, this means that we'll use
  40  * 2 memory pages and waste 4092 bytes of the second page that will never be
  41  * used.
  42  */
  43 #define HOST1X_PUSHBUFFER_SLOTS 511
  44
  45 /*
  46  * Clean up push buffer resources
  47  */
  48 static void host1x_pushbuffer_destroy(struct push_buffer *pb)
  49 {
  50         struct host1x_cdma *cdma = pb_to_cdma(pb);
  51         struct host1x *host1x = cdma_to_host1x(cdma);
  52
  53         if (!pb->mapped)
  54                 return;
  55
  56         if (host1x->domain) {
  57                 iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);
  58                 free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));
  59         }
  60
  61         dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);
  62
  63         pb->mapped = NULL;
  64         pb->phys = 0;
  65 }
  66
  67 /*
  68  * Init push buffer resources
  69  */
  70 static int host1x_pushbuffer_init(struct push_buffer *pb)
  71 {
  72         struct host1x_cdma *cdma = pb_to_cdma(pb);
  73         struct host1x *host1x = cdma_to_host1x(cdma);
  74         struct iova *alloc;
  75         u32 size;
  76         int err;
  77
  78         pb->mapped = NULL;
  79         pb->phys = 0;
  80         pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;
  81
  82         size = pb->size + 4;
  83
  84         /* initialize buffer pointers */
  85         pb->fence = pb->size - 8;
  86         pb->pos = 0;
  87
  88         if (host1x->domain) {
  89                 unsigned long shift;
  90
  91                 size = iova_align(&host1x->iova, size);
  92
  93                 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
  94                                           GFP_KERNEL);
  95                 if (!pb->mapped)
  96                         return -ENOMEM;
  97
  98                 shift = iova_shift(&host1x->iova);
  99                 alloc = alloc_iova(&host1x->iova, size >> shift,
 100                                    host1x->iova_end >> shift, true);
 101                 if (!alloc) {
 102                         err = -ENOMEM;
 103                         goto iommu_free_mem;
 104                 }
 105
 106                 pb->dma = iova_dma_addr(&host1x->iova, alloc);
 107                 err = iommu_map(host1x->domain, pb->dma, pb->phys, size,
 108                                 IOMMU_READ);
 109                 if (err)
 110                         goto iommu_free_iova;
 111         } else {
 112                 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
 113                                           GFP_KERNEL);
 114                 if (!pb->mapped)
 115                         return -ENOMEM;
 116
 117                 pb->dma = pb->phys;
 118         }
 119
 120         pb->alloc_size = size;
 121
 122         host1x_hw_pushbuffer_init(host1x, pb);
 123
 124         return 0;
 125
 126 iommu_free_iova:
 127         __free_iova(&host1x->iova, alloc);
 128 iommu_free_mem:
 129         dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);
 130
 131         return err;
 132 }
 133
 134 /*
 135  * Push two words to the push buffer
 136  * Caller must ensure push buffer is not full
 137  */
 138 static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
 139 {
 140         u32 *p = (u32 *)((void *)pb->mapped + pb->pos);
 141
 142         WARN_ON(pb->pos == pb->fence);
 143         *(p++) = op1;
 144         *(p++) = op2;
 145         pb->pos += 8;
 146
 147         if (pb->pos >= pb->size)
 148                 pb->pos -= pb->size;
 149 }
 150
 151 /*
 152  * Pop a number of two word slots from the push buffer
 153  * Caller must ensure push buffer is not empty
 154  */
 155 static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
 156 {
 157         /* Advance the next write position */
 158         pb->fence += slots * 8;
 159
 160         if (pb->fence >= pb->size)
 161                 pb->fence -= pb->size;
 162 }
 163
 164 /*
 165  * Return the number of two word slots free in the push buffer
 166  */
 167 static u32 host1x_pushbuffer_space(struct push_buffer *pb)
 168 {
 169         unsigned int fence = pb->fence;
 170
 171         if (pb->fence < pb->pos)
 172                 fence += pb->size;
 173
 174         return (fence - pb->pos) / 8;
 175 }
 176
 177 /*
 178  * Sleep (if necessary) until the requested event happens
 179  *   - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
 180  *     - Returns 1
 181  *   - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
 182  *     - Return the amount of space (> 0)
 183  * Must be called with the cdma lock held.
 184  */
 185 unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
 186                                      enum cdma_event event)
 187 {
 188         for (;;) {
 189                 struct push_buffer *pb = &cdma->push_buffer;
 190                 unsigned int space;
 191
 192                 switch (event) {
 193                 case CDMA_EVENT_SYNC_QUEUE_EMPTY:
 194                         space = list_empty(&cdma->sync_queue) ? 1 : 0;
 195                         break;
 196
 197                 case CDMA_EVENT_PUSH_BUFFER_SPACE:
 198                         space = host1x_pushbuffer_space(pb);
 199                         break;
 200
 201                 default:
 202                         WARN_ON(1);
 203                         return -EINVAL;
 204                 }
 205
 206                 if (space)
 207                         return space;
 208
 209                 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
 210                                        event);
 211
 212                 /* If somebody has managed to already start waiting, yield */
 213                 if (cdma->event != CDMA_EVENT_NONE) {
 214                         mutex_unlock(&cdma->lock);
 215                         schedule();
 216                         mutex_lock(&cdma->lock);
 217                         continue;
 218                 }
 219
 220                 cdma->event = event;
 221
 222                 mutex_unlock(&cdma->lock);
 223                 wait_for_completion(&cdma->complete);
 224                 mutex_lock(&cdma->lock);
 225         }
 226
 227         return 0;
 228 }
 229
 230 /*
 231  * Sleep (if necessary) until the push buffer has enough free space.
 232  *
 233  * Must be called with the cdma lock held.
 234  */
 235 static int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x,
 236                                              struct host1x_cdma *cdma,
 237                                              unsigned int needed)
 238 {
 239         while (true) {
 240                 struct push_buffer *pb = &cdma->push_buffer;
 241                 unsigned int space;
 242
 243                 space = host1x_pushbuffer_space(pb);
 244                 if (space >= needed)
 245                         break;
 246
 247                 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
 248                                        CDMA_EVENT_PUSH_BUFFER_SPACE);
 249
 250                 host1x_hw_cdma_flush(host1x, cdma);
 251
 252                 /* If somebody has managed to already start waiting, yield */
 253                 if (cdma->event != CDMA_EVENT_NONE) {
 254                         mutex_unlock(&cdma->lock);
 255                         schedule();
 256                         mutex_lock(&cdma->lock);
 257                         continue;
 258                 }
 259
 260                 cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE;
 261
 262                 mutex_unlock(&cdma->lock);
 263                 wait_for_completion(&cdma->complete);
 264                 mutex_lock(&cdma->lock);
 265         }
 266
 267         return 0;
 268 }
 269 /*
 270  * Start timer that tracks the time spent by the job.
 271  * Must be called with the cdma lock held.
 272  */
 273 static void cdma_start_timer_locked(struct host1x_cdma *cdma,
 274                                     struct host1x_job *job)
 275 {
 276         if (cdma->timeout.client) {
 277                 /* timer already started */
 278                 return;
 279         }
 280
 281         cdma->timeout.client = job->client;
 282         cdma->timeout.syncpt = job->syncpt;
 283         cdma->timeout.syncpt_val = job->syncpt_end;
 284         cdma->timeout.start_ktime = ktime_get();
 285
 286         schedule_delayed_work(&cdma->timeout.wq,
 287                               msecs_to_jiffies(job->timeout));
 288 }
 289
 290 /*
 291  * Stop timer when a buffer submission completes.
 292  * Must be called with the cdma lock held.
 293  */
 294 static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
 295 {
 296         cancel_delayed_work(&cdma->timeout.wq);
 297         cdma->timeout.client = NULL;
 298 }
 299
 300 /*
 301  * For all sync queue entries that have already finished according to the
 302  * current sync point registers:
 303  *  - unpin & unref their mems
 304  *  - pop their push buffer slots
 305  *  - remove them from the sync queue
 306  * This is normally called from the host code's worker thread, but can be
 307  * called manually if necessary.
 308  * Must be called with the cdma lock held.
 309  */
 310 static void update_cdma_locked(struct host1x_cdma *cdma)
 311 {
 312         bool signal = false;
 313         struct host1x_job *job, *n;
 314
 315         /*
 316          * Walk the sync queue, reading the sync point registers as necessary,
 317          * to consume as many sync queue entries as possible without blocking
 318          */
 319         list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
 320                 struct host1x_syncpt *sp = job->syncpt;
 321
 322                 /* Check whether this syncpt has completed, and bail if not */
 323                 if (!host1x_syncpt_is_expired(sp, job->syncpt_end) &&
 324                     !job->cancelled) {
 325                         /* Start timer on next pending syncpt */
 326                         if (job->timeout)
 327                                 cdma_start_timer_locked(cdma, job);
 328
 329                         break;
 330                 }
 331
 332                 /* Cancel timeout, when a buffer completes */
 333                 if (cdma->timeout.client)
 334                         stop_cdma_timer_locked(cdma);
 335
 336                 /* Unpin the memory */
 337                 host1x_job_unpin(job);
 338
 339                 /* Pop push buffer slots */
 340                 if (job->num_slots) {
 341                         struct push_buffer *pb = &cdma->push_buffer;
 342
 343                         host1x_pushbuffer_pop(pb, job->num_slots);
 344
 345                         if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
 346                                 signal = true;
 347                 }
 348
 349                 list_del(&job->list);
 350                 host1x_job_put(job);
 351         }
 352
 353         if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
 354             list_empty(&cdma->sync_queue))
 355                 signal = true;
 356
 357         if (signal) {
 358                 cdma->event = CDMA_EVENT_NONE;
 359                 complete(&cdma->complete);
 360         }
 361 }
 362
 363 void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
 364                                    struct device *dev)
 365 {
 366         struct host1x *host1x = cdma_to_host1x(cdma);
 367         u32 restart_addr, syncpt_incrs, syncpt_val;
 368         struct host1x_job *job, *next_job = NULL;
 369
 370         syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 371
 372         dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
 373                 __func__, syncpt_val);
 374
 375         /*
 376          * Move the sync_queue read pointer to the first entry that hasn't
 377          * completed based on the current HW syncpt value. It's likely there
 378          * won't be any (i.e. we're still at the head), but covers the case
 379          * where a syncpt incr happens just prior/during the teardown.
 380          */
 381
 382         dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
 383                 __func__);
 384
 385         list_for_each_entry(job, &cdma->sync_queue, list) {
 386                 if (syncpt_val < job->syncpt_end) {
 387
 388                         if (!list_is_last(&job->list, &cdma->sync_queue))
 389                                 next_job = list_next_entry(job, list);
 390
 391                         goto syncpt_incr;
 392                 }
 393
 394                 host1x_job_dump(dev, job);
 395         }
 396
 397         /* all jobs have been completed */
 398         job = NULL;
 399
 400 syncpt_incr:
 401
 402         /*
 403          * Increment with CPU the remaining syncpts of a partially executed job.
 404          *
 405          * CDMA will continue execution starting with the next job or will get
 406          * into idle state.
 407          */
 408         if (next_job)
 409                 restart_addr = next_job->first_get;
 410         else
 411                 restart_addr = cdma->last_pos;
 412
 413         if (!job)
 414                 goto resume;
 415
 416         /* do CPU increments for the remaining syncpts */
 417         if (job->syncpt_recovery) {
 418                 dev_dbg(dev, "%s: perform CPU incr on pending buffers\n",
 419                         __func__);
 420
 421                 /* won't need a timeout when replayed */
 422                 job->timeout = 0;
 423
 424                 syncpt_incrs = job->syncpt_end - syncpt_val;
 425                 dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
 426
 427                 host1x_job_dump(dev, job);
 428
 429                 /* safe to use CPU to incr syncpts */
 430                 host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
 431                                                 syncpt_incrs, job->syncpt_end,
 432                                                 job->num_slots);
 433
 434                 dev_dbg(dev, "%s: finished sync_queue modification\n",
 435                         __func__);
 436         } else {
 437                 struct host1x_job *failed_job = job;
 438
 439                 host1x_job_dump(dev, job);
 440
 441                 host1x_syncpt_set_locked(job->syncpt);
 442                 failed_job->cancelled = true;
 443
 444                 list_for_each_entry_continue(job, &cdma->sync_queue, list) {
 445                         unsigned int i;
 446
 447                         if (job->syncpt != failed_job->syncpt)
 448                                 continue;
 449
 450                         for (i = 0; i < job->num_slots; i++) {
 451                                 unsigned int slot = (job->first_get/8 + i) %
 452                                                     HOST1X_PUSHBUFFER_SLOTS;
 453                                 u32 *mapped = cdma->push_buffer.mapped;
 454
 455                                 /*
 456                                  * Overwrite opcodes with 0 word writes
 457                                  * to offset 0xbad. This does nothing but
 458                                  * has a easily detected signature in debug
 459                                  * traces.
 460                                  *
 461                                  * On systems with MLOCK enforcement enabled,
 462                                  * the above 0 word writes would fall foul of
 463                                  * the enforcement. As such, in the first slot
 464                                  * put a RESTART_W opcode to the beginning
 465                                  * of the next job. We don't use this for older
 466                                  * chips since those only support the RESTART
 467                                  * opcode with inconvenient alignment requirements.
 468                                  */
 469                                 if (i == 0 && host1x->info->has_wide_gather) {
 470                                         unsigned int next_job = (job->first_get/8 + job->num_slots)
 471                                                 % HOST1X_PUSHBUFFER_SLOTS;
 472                                         mapped[2*slot+0] = (0xd << 28) | (next_job * 2);
 473                                         mapped[2*slot+1] = 0x0;
 474                                 } else {
 475                                         mapped[2*slot+0] = 0x1bad0000;
 476                                         mapped[2*slot+1] = 0x1bad0000;
 477                                 }
 478                         }
 479
 480                         job->cancelled = true;
 481                 }
 482
 483                 wmb();
 484
 485                 update_cdma_locked(cdma);
 486         }
 487
 488 resume:
 489         /* roll back DMAGET and start up channel again */
 490         host1x_hw_cdma_resume(host1x, cdma, restart_addr);
 491 }
 492
 493 /*
 494  * Create a cdma
 495  */
 496 int host1x_cdma_init(struct host1x_cdma *cdma)
 497 {
 498         int err;
 499
 500         mutex_init(&cdma->lock);
 501         init_completion(&cdma->complete);
 502
 503         INIT_LIST_HEAD(&cdma->sync_queue);
 504
 505         cdma->event = CDMA_EVENT_NONE;
 506         cdma->running = false;
 507         cdma->torndown = false;
 508
 509         err = host1x_pushbuffer_init(&cdma->push_buffer);
 510         if (err)
 511                 return err;
 512
 513         return 0;
 514 }
 515
 516 /*
 517  * Destroy a cdma
 518  */
 519 int host1x_cdma_deinit(struct host1x_cdma *cdma)
 520 {
 521         struct push_buffer *pb = &cdma->push_buffer;
 522         struct host1x *host1x = cdma_to_host1x(cdma);
 523
 524         if (cdma->running) {
 525                 pr_warn("%s: CDMA still running\n", __func__);
 526                 return -EBUSY;
 527         }
 528
 529         host1x_pushbuffer_destroy(pb);
 530         host1x_hw_cdma_timeout_destroy(host1x, cdma);
 531
 532         return 0;
 533 }
 534
 535 /*
 536  * Begin a cdma submit
 537  */
 538 int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
 539 {
 540         struct host1x *host1x = cdma_to_host1x(cdma);
 541
 542         mutex_lock(&cdma->lock);
 543
 544         /*
 545          * Check if syncpoint was locked due to previous job timeout.
 546          * This needs to be done within the cdma lock to avoid a race
 547          * with the timeout handler.
 548          */
 549         if (job->syncpt->locked) {
 550                 mutex_unlock(&cdma->lock);
 551                 return -EPERM;
 552         }
 553
 554         if (job->timeout) {
 555                 /* init state on first submit with timeout value */
 556                 if (!cdma->timeout.initialized) {
 557                         int err;
 558
 559                         err = host1x_hw_cdma_timeout_init(host1x, cdma);
 560                         if (err) {
 561                                 mutex_unlock(&cdma->lock);
 562                                 return err;
 563                         }
 564                 }
 565         }
 566
 567         if (!cdma->running)
 568                 host1x_hw_cdma_start(host1x, cdma);
 569
 570         cdma->slots_free = 0;
 571         cdma->slots_used = 0;
 572         cdma->first_get = cdma->push_buffer.pos;
 573
 574         trace_host1x_cdma_begin(dev_name(job->channel->dev));
 575         return 0;
 576 }
 577
 578 /*
 579  * Push two words into a push buffer slot
 580  * Blocks as necessary if the push buffer is full.
 581  */
 582 void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
 583 {
 584         struct host1x *host1x = cdma_to_host1x(cdma);
 585         struct push_buffer *pb = &cdma->push_buffer;
 586         u32 slots_free = cdma->slots_free;
 587
 588         if (host1x_debug_trace_cmdbuf)
 589                 trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
 590                                        op1, op2);
 591
 592         if (slots_free == 0) {
 593                 host1x_hw_cdma_flush(host1x, cdma);
 594                 slots_free = host1x_cdma_wait_locked(cdma,
 595                                                 CDMA_EVENT_PUSH_BUFFER_SPACE);
 596         }
 597
 598         cdma->slots_free = slots_free - 1;
 599         cdma->slots_used++;
 600         host1x_pushbuffer_push(pb, op1, op2);
 601 }
 602
 603 /*
 604  * Push four words into two consecutive push buffer slots. Note that extra
 605  * care needs to be taken not to split the two slots across the end of the
 606  * push buffer. Otherwise the RESTART opcode at the end of the push buffer
 607  * that ensures processing will restart at the beginning will break up the
 608  * four words.
 609  *
 610  * Blocks as necessary if the push buffer is full.
 611  */
 612 void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2,
 613                            u32 op3, u32 op4)
 614 {
 615         struct host1x_channel *channel = cdma_to_channel(cdma);
 616         struct host1x *host1x = cdma_to_host1x(cdma);
 617         struct push_buffer *pb = &cdma->push_buffer;
 618         unsigned int space = cdma->slots_free;
 619         unsigned int needed = 2, extra = 0;
 620
 621         if (host1x_debug_trace_cmdbuf)
 622                 trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2,
 623                                             op3, op4);
 624
 625         /* compute number of extra slots needed for padding */
 626         if (pb->pos + 16 > pb->size) {
 627                 extra = (pb->size - pb->pos) / 8;
 628                 needed += extra;
 629         }
 630
 631         host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed);
 632         space = host1x_pushbuffer_space(pb);
 633
 634         cdma->slots_free = space - needed;
 635         cdma->slots_used += needed;
 636
 637         if (extra > 0) {
 638                 /*
 639                  * If there isn't enough space at the tail of the pushbuffer,
 640                  * insert a RESTART(0) here to go back to the beginning.
 641                  * The code above adjusted the indexes appropriately.
 642                  */
 643                 host1x_pushbuffer_push(pb, (0x5 << 28), 0xdead0000);
 644         }
 645
 646         host1x_pushbuffer_push(pb, op1, op2);
 647         host1x_pushbuffer_push(pb, op3, op4);
 648 }
 649
 650 /*
 651  * End a cdma submit
 652  * Kick off DMA, add job to the sync queue, and a number of slots to be freed
 653  * from the pushbuffer. The handles for a submit must all be pinned at the same
 654  * time, but they can be unpinned in smaller chunks.
 655  */
 656 void host1x_cdma_end(struct host1x_cdma *cdma,
 657                      struct host1x_job *job)
 658 {
 659         struct host1x *host1x = cdma_to_host1x(cdma);
 660         bool idle = list_empty(&cdma->sync_queue);
 661
 662         host1x_hw_cdma_flush(host1x, cdma);
 663
 664         job->first_get = cdma->first_get;
 665         job->num_slots = cdma->slots_used;
 666         host1x_job_get(job);
 667         list_add_tail(&job->list, &cdma->sync_queue);
 668
 669         /* start timer on idle -> active transitions */
 670         if (job->timeout && idle)
 671                 cdma_start_timer_locked(cdma, job);
 672
 673         trace_host1x_cdma_end(dev_name(job->channel->dev));
 674         mutex_unlock(&cdma->lock);
 675 }
 676
 677 /*
 678  * Update cdma state according to current sync point values
 679  */
 680 void host1x_cdma_update(struct host1x_cdma *cdma)
 681 {
 682         mutex_lock(&cdma->lock);
 683         update_cdma_locked(cdma);
 684         mutex_unlock(&cdma->lock);
 685 }