drivers/accel/ivpu/ivpu_job.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2023 Intel Corporation
   4  */
   5
   6 #include <drm/drm_file.h>
   7
   8 #include <linux/bitfield.h>
   9 #include <linux/highmem.h>
  10 #include <linux/pci.h>
  11 #include <linux/module.h>
  12 #include <uapi/drm/ivpu_accel.h>
  13
  14 #include "ivpu_drv.h"
  15 #include "ivpu_hw.h"
  16 #include "ivpu_ipc.h"
  17 #include "ivpu_job.h"
  18 #include "ivpu_jsm_msg.h"
  19 #include "ivpu_pm.h"
  20
  21 #define CMD_BUF_IDX          0
  22 #define JOB_ID_JOB_MASK      GENMASK(7, 0)
  23 #define JOB_ID_CONTEXT_MASK  GENMASK(31, 8)
  24 #define JOB_MAX_BUFFER_COUNT 65535
  25
  26 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)
  27 {
  28         ivpu_hw_reg_db_set(vdev, cmdq->db_id);
  29 }
  30
  31 static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv, u16 engine)
  32 {
  33         struct ivpu_device *vdev = file_priv->vdev;
  34         struct vpu_job_queue_header *jobq_header;
  35         struct ivpu_cmdq *cmdq;
  36
  37         cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL);
  38         if (!cmdq)
  39                 return NULL;
  40
  41         cmdq->mem = ivpu_bo_alloc_internal(vdev, 0, SZ_4K, DRM_IVPU_BO_WC);
  42         if (!cmdq->mem)
  43                 goto cmdq_free;
  44
  45         cmdq->db_id = file_priv->ctx.id + engine * ivpu_get_context_count(vdev);
  46         cmdq->entry_count = (u32)((ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header)) /
  47                                   sizeof(struct vpu_job_queue_entry));
  48
  49         cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem);
  50         jobq_header = &cmdq->jobq->header;
  51         jobq_header->engine_idx = engine;
  52         jobq_header->head = 0;
  53         jobq_header->tail = 0;
  54         wmb(); /* Flush WC buffer for jobq->header */
  55
  56         return cmdq;
  57
  58 cmdq_free:
  59         kfree(cmdq);
  60         return NULL;
  61 }
  62
  63 static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
  64 {
  65         if (!cmdq)
  66                 return;
  67
  68         ivpu_bo_free_internal(cmdq->mem);
  69         kfree(cmdq);
  70 }
  71
  72 static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u16 engine)
  73 {
  74         struct ivpu_device *vdev = file_priv->vdev;
  75         struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
  76         int ret;
  77
  78         lockdep_assert_held(&file_priv->lock);
  79
  80         if (!cmdq) {
  81                 cmdq = ivpu_cmdq_alloc(file_priv, engine);
  82                 if (!cmdq)
  83                         return NULL;
  84                 file_priv->cmdq[engine] = cmdq;
  85         }
  86
  87         if (cmdq->db_registered)
  88                 return cmdq;
  89
  90         ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,
  91                                    cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
  92         if (ret)
  93                 return NULL;
  94
  95         cmdq->db_registered = true;
  96
  97         return cmdq;
  98 }
  99
 100 static void ivpu_cmdq_release_locked(struct ivpu_file_priv *file_priv, u16 engine)
 101 {
 102         struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
 103
 104         lockdep_assert_held(&file_priv->lock);
 105
 106         if (cmdq) {
 107                 file_priv->cmdq[engine] = NULL;
 108                 if (cmdq->db_registered)
 109                         ivpu_jsm_unregister_db(file_priv->vdev, cmdq->db_id);
 110
 111                 ivpu_cmdq_free(file_priv, cmdq);
 112         }
 113 }
 114
 115 void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv)
 116 {
 117         int i;
 118
 119         lockdep_assert_held(&file_priv->lock);
 120
 121         for (i = 0; i < IVPU_NUM_ENGINES; i++)
 122                 ivpu_cmdq_release_locked(file_priv, i);
 123 }
 124
 125 /*
 126  * Mark the doorbell as unregistered and reset job queue pointers.
 127  * This function needs to be called when the VPU hardware is restarted
 128  * and FW loses job queue state. The next time job queue is used it
 129  * will be registered again.
 130  */
 131 static void ivpu_cmdq_reset_locked(struct ivpu_file_priv *file_priv, u16 engine)
 132 {
 133         struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
 134
 135         lockdep_assert_held(&file_priv->lock);
 136
 137         if (cmdq) {
 138                 cmdq->db_registered = false;
 139                 cmdq->jobq->header.head = 0;
 140                 cmdq->jobq->header.tail = 0;
 141                 wmb(); /* Flush WC buffer for jobq header */
 142         }
 143 }
 144
 145 static void ivpu_cmdq_reset_all(struct ivpu_file_priv *file_priv)
 146 {
 147         int i;
 148
 149         mutex_lock(&file_priv->lock);
 150
 151         for (i = 0; i < IVPU_NUM_ENGINES; i++)
 152                 ivpu_cmdq_reset_locked(file_priv, i);
 153
 154         mutex_unlock(&file_priv->lock);
 155 }
 156
 157 void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev)
 158 {
 159         struct ivpu_file_priv *file_priv;
 160         unsigned long ctx_id;
 161
 162         mutex_lock(&vdev->context_list_lock);
 163
 164         xa_for_each(&vdev->context_xa, ctx_id, file_priv)
 165                 ivpu_cmdq_reset_all(file_priv);
 166
 167         mutex_unlock(&vdev->context_list_lock);
 168
 169 }
 170
 171 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)
 172 {
 173         struct ivpu_device *vdev = job->vdev;
 174         struct vpu_job_queue_header *header = &cmdq->jobq->header;
 175         struct vpu_job_queue_entry *entry;
 176         u32 tail = READ_ONCE(header->tail);
 177         u32 next_entry = (tail + 1) % cmdq->entry_count;
 178
 179         /* Check if there is space left in job queue */
 180         if (next_entry == header->head) {
 181                 ivpu_dbg(vdev, JOB, "Job queue full: ctx %d engine %d db %d head %d tail %d\n",
 182                          job->file_priv->ctx.id, job->engine_idx, cmdq->db_id, header->head, tail);
 183                 return -EBUSY;
 184         }
 185
 186         entry = &cmdq->jobq->job[tail];
 187         entry->batch_buf_addr = job->cmd_buf_vpu_addr;
 188         entry->job_id = job->job_id;
 189         entry->flags = 0;
 190         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION))
 191                 entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK;
 192         wmb(); /* Ensure that tail is updated after filling entry */
 193         header->tail = next_entry;
 194         wmb(); /* Flush WC buffer for jobq header */
 195
 196         return 0;
 197 }
 198
 199 struct ivpu_fence {
 200         struct dma_fence base;
 201         spinlock_t lock; /* protects base */
 202         struct ivpu_device *vdev;
 203 };
 204
 205 static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence)
 206 {
 207         return container_of(fence, struct ivpu_fence, base);
 208 }
 209
 210 static const char *ivpu_fence_get_driver_name(struct dma_fence *fence)
 211 {
 212         return DRIVER_NAME;
 213 }
 214
 215 static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence)
 216 {
 217         struct ivpu_fence *ivpu_fence = to_vpu_fence(fence);
 218
 219         return dev_name(ivpu_fence->vdev->drm.dev);
 220 }
 221
 222 static const struct dma_fence_ops ivpu_fence_ops = {
 223         .get_driver_name = ivpu_fence_get_driver_name,
 224         .get_timeline_name = ivpu_fence_get_timeline_name,
 225 };
 226
 227 static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev)
 228 {
 229         struct ivpu_fence *fence;
 230
 231         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 232         if (!fence)
 233                 return NULL;
 234
 235         fence->vdev = vdev;
 236         spin_lock_init(&fence->lock);
 237         dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1);
 238
 239         return &fence->base;
 240 }
 241
 242 static void ivpu_job_destroy(struct ivpu_job *job)
 243 {
 244         struct ivpu_device *vdev = job->vdev;
 245         u32 i;
 246
 247         ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d engine %d",
 248                  job->job_id, job->file_priv->ctx.id, job->engine_idx);
 249
 250         for (i = 0; i < job->bo_count; i++)
 251                 if (job->bos[i])
 252                         drm_gem_object_put(&job->bos[i]->base.base);
 253
 254         dma_fence_put(job->done_fence);
 255         ivpu_file_priv_put(&job->file_priv);
 256         kfree(job);
 257 }
 258
 259 static struct ivpu_job *
 260 ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)
 261 {
 262         struct ivpu_device *vdev = file_priv->vdev;
 263         struct ivpu_job *job;
 264
 265         job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL);
 266         if (!job)
 267                 return NULL;
 268
 269         job->vdev = vdev;
 270         job->engine_idx = engine_idx;
 271         job->bo_count = bo_count;
 272         job->done_fence = ivpu_fence_create(vdev);
 273         if (!job->done_fence) {
 274                 ivpu_warn_ratelimited(vdev, "Failed to create a fence\n");
 275                 goto err_free_job;
 276         }
 277
 278         job->file_priv = ivpu_file_priv_get(file_priv);
 279
 280         ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);
 281         return job;
 282
 283 err_free_job:
 284         kfree(job);
 285         return NULL;
 286 }
 287
 288 static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status)
 289 {
 290         struct ivpu_job *job;
 291
 292         job = xa_erase(&vdev->submitted_jobs_xa, job_id);
 293         if (!job)
 294                 return -ENOENT;
 295
 296         if (job->file_priv->has_mmu_faults)
 297                 job_status = DRM_IVPU_JOB_STATUS_ABORTED;
 298
 299         job->bos[CMD_BUF_IDX]->job_status = job_status;
 300         dma_fence_signal(job->done_fence);
 301
 302         ivpu_dbg(vdev, JOB, "Job complete:  id %3u ctx %2d engine %d status 0x%x\n",
 303                  job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);
 304
 305         ivpu_job_destroy(job);
 306         ivpu_stop_job_timeout_detection(vdev);
 307
 308         ivpu_rpm_put(vdev);
 309         return 0;
 310 }
 311
 312 void ivpu_jobs_abort_all(struct ivpu_device *vdev)
 313 {
 314         struct ivpu_job *job;
 315         unsigned long id;
 316
 317         xa_for_each(&vdev->submitted_jobs_xa, id, job)
 318                 ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED);
 319 }
 320
 321 static int ivpu_job_submit(struct ivpu_job *job)
 322 {
 323         struct ivpu_file_priv *file_priv = job->file_priv;
 324         struct ivpu_device *vdev = job->vdev;
 325         struct xa_limit job_id_range;
 326         struct ivpu_cmdq *cmdq;
 327         int ret;
 328
 329         ret = ivpu_rpm_get(vdev);
 330         if (ret < 0)
 331                 return ret;
 332
 333         mutex_lock(&file_priv->lock);
 334
 335         cmdq = ivpu_cmdq_acquire(job->file_priv, job->engine_idx);
 336         if (!cmdq) {
 337                 ivpu_warn_ratelimited(vdev, "Failed get job queue, ctx %d engine %d\n",
 338                                       file_priv->ctx.id, job->engine_idx);
 339                 ret = -EINVAL;
 340                 goto err_unlock_file_priv;
 341         }
 342
 343         job_id_range.min = FIELD_PREP(JOB_ID_CONTEXT_MASK, (file_priv->ctx.id - 1));
 344         job_id_range.max = job_id_range.min | JOB_ID_JOB_MASK;
 345
 346         xa_lock(&vdev->submitted_jobs_xa);
 347         ret = __xa_alloc(&vdev->submitted_jobs_xa, &job->job_id, job, job_id_range, GFP_KERNEL);
 348         if (ret) {
 349                 ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n",
 350                          file_priv->ctx.id);
 351                 ret = -EBUSY;
 352                 goto err_unlock_submitted_jobs_xa;
 353         }
 354
 355         ret = ivpu_cmdq_push_job(cmdq, job);
 356         if (ret)
 357                 goto err_erase_xa;
 358
 359         ivpu_start_job_timeout_detection(vdev);
 360
 361         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) {
 362                 cmdq->jobq->header.head = cmdq->jobq->header.tail;
 363                 wmb(); /* Flush WC buffer for jobq header */
 364         } else {
 365                 ivpu_cmdq_ring_db(vdev, cmdq);
 366         }
 367
 368         ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d engine %d addr 0x%llx next %d\n",
 369                  job->job_id, file_priv->ctx.id, job->engine_idx,
 370                  job->cmd_buf_vpu_addr, cmdq->jobq->header.tail);
 371
 372         xa_unlock(&vdev->submitted_jobs_xa);
 373
 374         mutex_unlock(&file_priv->lock);
 375
 376         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW))
 377                 ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);
 378
 379         return 0;
 380
 381 err_erase_xa:
 382         __xa_erase(&vdev->submitted_jobs_xa, job->job_id);
 383 err_unlock_submitted_jobs_xa:
 384         xa_unlock(&vdev->submitted_jobs_xa);
 385 err_unlock_file_priv:
 386         mutex_unlock(&file_priv->lock);
 387         ivpu_rpm_put(vdev);
 388         return ret;
 389 }
 390
 391 static int
 392 ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles,
 393                                 u32 buf_count, u32 commands_offset)
 394 {
 395         struct ivpu_file_priv *file_priv = file->driver_priv;
 396         struct ivpu_device *vdev = file_priv->vdev;
 397         struct ww_acquire_ctx acquire_ctx;
 398         enum dma_resv_usage usage;
 399         struct ivpu_bo *bo;
 400         int ret;
 401         u32 i;
 402
 403         for (i = 0; i < buf_count; i++) {
 404                 struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]);
 405
 406                 if (!obj)
 407                         return -ENOENT;
 408
 409                 job->bos[i] = to_ivpu_bo(obj);
 410
 411                 ret = ivpu_bo_pin(job->bos[i]);
 412                 if (ret)
 413                         return ret;
 414         }
 415
 416         bo = job->bos[CMD_BUF_IDX];
 417         if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) {
 418                 ivpu_warn(vdev, "Buffer is already in use\n");
 419                 return -EBUSY;
 420         }
 421
 422         if (commands_offset >= ivpu_bo_size(bo)) {
 423                 ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset);
 424                 return -EINVAL;
 425         }
 426
 427         job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;
 428
 429         ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,
 430                                         &acquire_ctx);
 431         if (ret) {
 432                 ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret);
 433                 return ret;
 434         }
 435
 436         for (i = 0; i < buf_count; i++) {
 437                 ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1);
 438                 if (ret) {
 439                         ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);
 440                         goto unlock_reservations;
 441                 }
 442         }
 443
 444         for (i = 0; i < buf_count; i++) {
 445                 usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP;
 446                 dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage);
 447         }
 448
 449 unlock_reservations:
 450         drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);
 451
 452         wmb(); /* Flush write combining buffers */
 453
 454         return ret;
 455 }
 456
 457 int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 458 {
 459         struct ivpu_file_priv *file_priv = file->driver_priv;
 460         struct ivpu_device *vdev = file_priv->vdev;
 461         struct drm_ivpu_submit *params = data;
 462         struct ivpu_job *job;
 463         u32 *buf_handles;
 464         int idx, ret;
 465
 466         if (params->engine > DRM_IVPU_ENGINE_COPY)
 467                 return -EINVAL;
 468
 469         if (params->priority > DRM_IVPU_JOB_PRIORITY_REALTIME)
 470                 return -EINVAL;
 471
 472         if (params->buffer_count == 0 || params->buffer_count > JOB_MAX_BUFFER_COUNT)
 473                 return -EINVAL;
 474
 475         if (!IS_ALIGNED(params->commands_offset, 8))
 476                 return -EINVAL;
 477
 478         if (!file_priv->ctx.id)
 479                 return -EINVAL;
 480
 481         if (file_priv->has_mmu_faults)
 482                 return -EBADFD;
 483
 484         buf_handles = kcalloc(params->buffer_count, sizeof(u32), GFP_KERNEL);
 485         if (!buf_handles)
 486                 return -ENOMEM;
 487
 488         ret = copy_from_user(buf_handles,
 489                              (void __user *)params->buffers_ptr,
 490                              params->buffer_count * sizeof(u32));
 491         if (ret) {
 492                 ret = -EFAULT;
 493                 goto err_free_handles;
 494         }
 495
 496         if (!drm_dev_enter(&vdev->drm, &idx)) {
 497                 ret = -ENODEV;
 498                 goto err_free_handles;
 499         }
 500
 501         ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n",
 502                  file_priv->ctx.id, params->buffer_count);
 503
 504         job = ivpu_job_create(file_priv, params->engine, params->buffer_count);
 505         if (!job) {
 506                 ivpu_err(vdev, "Failed to create job\n");
 507                 ret = -ENOMEM;
 508                 goto err_exit_dev;
 509         }
 510
 511         ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count,
 512                                               params->commands_offset);
 513         if (ret) {
 514                 ivpu_err(vdev, "Failed to prepare job: %d\n", ret);
 515                 goto err_destroy_job;
 516         }
 517
 518         down_read(&vdev->pm->reset_lock);
 519         ret = ivpu_job_submit(job);
 520         up_read(&vdev->pm->reset_lock);
 521         if (ret)
 522                 goto err_signal_fence;
 523
 524         drm_dev_exit(idx);
 525         kfree(buf_handles);
 526         return ret;
 527
 528 err_signal_fence:
 529         dma_fence_signal(job->done_fence);
 530 err_destroy_job:
 531         ivpu_job_destroy(job);
 532 err_exit_dev:
 533         drm_dev_exit(idx);
 534 err_free_handles:
 535         kfree(buf_handles);
 536         return ret;
 537 }
 538
 539 static void
 540 ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
 541                        struct vpu_jsm_msg *jsm_msg)
 542 {
 543         struct vpu_ipc_msg_payload_job_done *payload;
 544         int ret;
 545
 546         if (!jsm_msg) {
 547                 ivpu_err(vdev, "IPC message has no JSM payload\n");
 548                 return;
 549         }
 550
 551         if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
 552                 ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result);
 553                 return;
 554         }
 555
 556         payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload;
 557         ret = ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status);
 558         if (!ret && !xa_empty(&vdev->submitted_jobs_xa))
 559                 ivpu_start_job_timeout_detection(vdev);
 560 }
 561
 562 void ivpu_job_done_consumer_init(struct ivpu_device *vdev)
 563 {
 564         ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer,
 565                               VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback);
 566 }
 567
 568 void ivpu_job_done_consumer_fini(struct ivpu_device *vdev)
 569 {
 570         ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer);
 571 }