1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2020-2023 Intel Corporation
6 #include <drm/drm_file.h>
8 #include <linux/bitfield.h>
9 #include <linux/highmem.h>
10 #include <linux/pci.h>
11 #include <linux/module.h>
12 #include <uapi/drm/ivpu_accel.h>
18 #include "ivpu_jsm_msg.h"
22 #define JOB_ID_JOB_MASK GENMASK(7, 0)
23 #define JOB_ID_CONTEXT_MASK GENMASK(31, 8)
24 #define JOB_MAX_BUFFER_COUNT 65535
26 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)
28 ivpu_hw_reg_db_set(vdev, cmdq->db_id);
31 static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv, u16 engine)
33 struct ivpu_device *vdev = file_priv->vdev;
34 struct vpu_job_queue_header *jobq_header;
35 struct ivpu_cmdq *cmdq;
37 cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL);
41 cmdq->mem = ivpu_bo_alloc_internal(vdev, 0, SZ_4K, DRM_IVPU_BO_WC);
45 cmdq->db_id = file_priv->ctx.id + engine * ivpu_get_context_count(vdev);
46 cmdq->entry_count = (u32)((ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header)) /
47 sizeof(struct vpu_job_queue_entry));
49 cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem);
50 jobq_header = &cmdq->jobq->header;
51 jobq_header->engine_idx = engine;
52 jobq_header->head = 0;
53 jobq_header->tail = 0;
54 wmb(); /* Flush WC buffer for jobq->header */
63 static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
68 ivpu_bo_free_internal(cmdq->mem);
72 static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u16 engine)
74 struct ivpu_device *vdev = file_priv->vdev;
75 struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
78 lockdep_assert_held(&file_priv->lock);
81 cmdq = ivpu_cmdq_alloc(file_priv, engine);
84 file_priv->cmdq[engine] = cmdq;
87 if (cmdq->db_registered)
90 ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,
91 cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
95 cmdq->db_registered = true;
100 static void ivpu_cmdq_release_locked(struct ivpu_file_priv *file_priv, u16 engine)
102 struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
104 lockdep_assert_held(&file_priv->lock);
107 file_priv->cmdq[engine] = NULL;
108 if (cmdq->db_registered)
109 ivpu_jsm_unregister_db(file_priv->vdev, cmdq->db_id);
111 ivpu_cmdq_free(file_priv, cmdq);
115 void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv)
119 lockdep_assert_held(&file_priv->lock);
121 for (i = 0; i < IVPU_NUM_ENGINES; i++)
122 ivpu_cmdq_release_locked(file_priv, i);
126 * Mark the doorbell as unregistered and reset job queue pointers.
127 * This function needs to be called when the VPU hardware is restarted
128 * and FW loses job queue state. The next time job queue is used it
129 * will be registered again.
131 static void ivpu_cmdq_reset_locked(struct ivpu_file_priv *file_priv, u16 engine)
133 struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
135 lockdep_assert_held(&file_priv->lock);
138 cmdq->db_registered = false;
139 cmdq->jobq->header.head = 0;
140 cmdq->jobq->header.tail = 0;
141 wmb(); /* Flush WC buffer for jobq header */
145 static void ivpu_cmdq_reset_all(struct ivpu_file_priv *file_priv)
149 mutex_lock(&file_priv->lock);
151 for (i = 0; i < IVPU_NUM_ENGINES; i++)
152 ivpu_cmdq_reset_locked(file_priv, i);
154 mutex_unlock(&file_priv->lock);
157 void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev)
159 struct ivpu_file_priv *file_priv;
160 unsigned long ctx_id;
162 mutex_lock(&vdev->context_list_lock);
164 xa_for_each(&vdev->context_xa, ctx_id, file_priv)
165 ivpu_cmdq_reset_all(file_priv);
167 mutex_unlock(&vdev->context_list_lock);
171 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)
173 struct ivpu_device *vdev = job->vdev;
174 struct vpu_job_queue_header *header = &cmdq->jobq->header;
175 struct vpu_job_queue_entry *entry;
176 u32 tail = READ_ONCE(header->tail);
177 u32 next_entry = (tail + 1) % cmdq->entry_count;
179 /* Check if there is space left in job queue */
180 if (next_entry == header->head) {
181 ivpu_dbg(vdev, JOB, "Job queue full: ctx %d engine %d db %d head %d tail %d\n",
182 job->file_priv->ctx.id, job->engine_idx, cmdq->db_id, header->head, tail);
186 entry = &cmdq->jobq->job[tail];
187 entry->batch_buf_addr = job->cmd_buf_vpu_addr;
188 entry->job_id = job->job_id;
190 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION))
191 entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK;
192 wmb(); /* Ensure that tail is updated after filling entry */
193 header->tail = next_entry;
194 wmb(); /* Flush WC buffer for jobq header */
200 struct dma_fence base;
201 spinlock_t lock; /* protects base */
202 struct ivpu_device *vdev;
205 static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence)
207 return container_of(fence, struct ivpu_fence, base);
210 static const char *ivpu_fence_get_driver_name(struct dma_fence *fence)
215 static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence)
217 struct ivpu_fence *ivpu_fence = to_vpu_fence(fence);
219 return dev_name(ivpu_fence->vdev->drm.dev);
222 static const struct dma_fence_ops ivpu_fence_ops = {
223 .get_driver_name = ivpu_fence_get_driver_name,
224 .get_timeline_name = ivpu_fence_get_timeline_name,
227 static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev)
229 struct ivpu_fence *fence;
231 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
236 spin_lock_init(&fence->lock);
237 dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1);
242 static void ivpu_job_destroy(struct ivpu_job *job)
244 struct ivpu_device *vdev = job->vdev;
247 ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d engine %d",
248 job->job_id, job->file_priv->ctx.id, job->engine_idx);
250 for (i = 0; i < job->bo_count; i++)
252 drm_gem_object_put(&job->bos[i]->base.base);
254 dma_fence_put(job->done_fence);
255 ivpu_file_priv_put(&job->file_priv);
259 static struct ivpu_job *
260 ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)
262 struct ivpu_device *vdev = file_priv->vdev;
263 struct ivpu_job *job;
265 job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL);
270 job->engine_idx = engine_idx;
271 job->bo_count = bo_count;
272 job->done_fence = ivpu_fence_create(vdev);
273 if (!job->done_fence) {
274 ivpu_warn_ratelimited(vdev, "Failed to create a fence\n");
278 job->file_priv = ivpu_file_priv_get(file_priv);
280 ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);
288 static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status)
290 struct ivpu_job *job;
292 job = xa_erase(&vdev->submitted_jobs_xa, job_id);
296 if (job->file_priv->has_mmu_faults)
297 job_status = DRM_IVPU_JOB_STATUS_ABORTED;
299 job->bos[CMD_BUF_IDX]->job_status = job_status;
300 dma_fence_signal(job->done_fence);
302 ivpu_dbg(vdev, JOB, "Job complete: id %3u ctx %2d engine %d status 0x%x\n",
303 job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);
305 ivpu_job_destroy(job);
306 ivpu_stop_job_timeout_detection(vdev);
312 void ivpu_jobs_abort_all(struct ivpu_device *vdev)
314 struct ivpu_job *job;
317 xa_for_each(&vdev->submitted_jobs_xa, id, job)
318 ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED);
321 static int ivpu_job_submit(struct ivpu_job *job)
323 struct ivpu_file_priv *file_priv = job->file_priv;
324 struct ivpu_device *vdev = job->vdev;
325 struct xa_limit job_id_range;
326 struct ivpu_cmdq *cmdq;
329 ret = ivpu_rpm_get(vdev);
333 mutex_lock(&file_priv->lock);
335 cmdq = ivpu_cmdq_acquire(job->file_priv, job->engine_idx);
337 ivpu_warn_ratelimited(vdev, "Failed get job queue, ctx %d engine %d\n",
338 file_priv->ctx.id, job->engine_idx);
340 goto err_unlock_file_priv;
343 job_id_range.min = FIELD_PREP(JOB_ID_CONTEXT_MASK, (file_priv->ctx.id - 1));
344 job_id_range.max = job_id_range.min | JOB_ID_JOB_MASK;
346 xa_lock(&vdev->submitted_jobs_xa);
347 ret = __xa_alloc(&vdev->submitted_jobs_xa, &job->job_id, job, job_id_range, GFP_KERNEL);
349 ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n",
352 goto err_unlock_submitted_jobs_xa;
355 ret = ivpu_cmdq_push_job(cmdq, job);
359 ivpu_start_job_timeout_detection(vdev);
361 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) {
362 cmdq->jobq->header.head = cmdq->jobq->header.tail;
363 wmb(); /* Flush WC buffer for jobq header */
365 ivpu_cmdq_ring_db(vdev, cmdq);
368 ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d engine %d addr 0x%llx next %d\n",
369 job->job_id, file_priv->ctx.id, job->engine_idx,
370 job->cmd_buf_vpu_addr, cmdq->jobq->header.tail);
372 xa_unlock(&vdev->submitted_jobs_xa);
374 mutex_unlock(&file_priv->lock);
376 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW))
377 ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);
382 __xa_erase(&vdev->submitted_jobs_xa, job->job_id);
383 err_unlock_submitted_jobs_xa:
384 xa_unlock(&vdev->submitted_jobs_xa);
385 err_unlock_file_priv:
386 mutex_unlock(&file_priv->lock);
392 ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles,
393 u32 buf_count, u32 commands_offset)
395 struct ivpu_file_priv *file_priv = file->driver_priv;
396 struct ivpu_device *vdev = file_priv->vdev;
397 struct ww_acquire_ctx acquire_ctx;
398 enum dma_resv_usage usage;
403 for (i = 0; i < buf_count; i++) {
404 struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]);
409 job->bos[i] = to_ivpu_bo(obj);
411 ret = ivpu_bo_pin(job->bos[i]);
416 bo = job->bos[CMD_BUF_IDX];
417 if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) {
418 ivpu_warn(vdev, "Buffer is already in use\n");
422 if (commands_offset >= ivpu_bo_size(bo)) {
423 ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset);
427 job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;
429 ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,
432 ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret);
436 for (i = 0; i < buf_count; i++) {
437 ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1);
439 ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);
440 goto unlock_reservations;
444 for (i = 0; i < buf_count; i++) {
445 usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP;
446 dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage);
450 drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);
452 wmb(); /* Flush write combining buffers */
457 int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
459 struct ivpu_file_priv *file_priv = file->driver_priv;
460 struct ivpu_device *vdev = file_priv->vdev;
461 struct drm_ivpu_submit *params = data;
462 struct ivpu_job *job;
466 if (params->engine > DRM_IVPU_ENGINE_COPY)
469 if (params->priority > DRM_IVPU_JOB_PRIORITY_REALTIME)
472 if (params->buffer_count == 0 || params->buffer_count > JOB_MAX_BUFFER_COUNT)
475 if (!IS_ALIGNED(params->commands_offset, 8))
478 if (!file_priv->ctx.id)
481 if (file_priv->has_mmu_faults)
484 buf_handles = kcalloc(params->buffer_count, sizeof(u32), GFP_KERNEL);
488 ret = copy_from_user(buf_handles,
489 (void __user *)params->buffers_ptr,
490 params->buffer_count * sizeof(u32));
493 goto err_free_handles;
496 if (!drm_dev_enter(&vdev->drm, &idx)) {
498 goto err_free_handles;
501 ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n",
502 file_priv->ctx.id, params->buffer_count);
504 job = ivpu_job_create(file_priv, params->engine, params->buffer_count);
506 ivpu_err(vdev, "Failed to create job\n");
511 ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count,
512 params->commands_offset);
514 ivpu_err(vdev, "Failed to prepare job: %d\n", ret);
515 goto err_destroy_job;
518 down_read(&vdev->pm->reset_lock);
519 ret = ivpu_job_submit(job);
520 up_read(&vdev->pm->reset_lock);
522 goto err_signal_fence;
529 dma_fence_signal(job->done_fence);
531 ivpu_job_destroy(job);
540 ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
541 struct vpu_jsm_msg *jsm_msg)
543 struct vpu_ipc_msg_payload_job_done *payload;
547 ivpu_err(vdev, "IPC message has no JSM payload\n");
551 if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
552 ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result);
556 payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload;
557 ret = ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status);
558 if (!ret && !xa_empty(&vdev->submitted_jobs_xa))
559 ivpu_start_job_timeout_detection(vdev);
562 void ivpu_job_done_consumer_init(struct ivpu_device *vdev)
564 ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer,
565 VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback);
568 void ivpu_job_done_consumer_fini(struct ivpu_device *vdev)
570 ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer);