1 // SPDX-License-Identifier: MIT
3 * Copyright © 2016-2019 Intel Corporation
7 #include "intel_guc_ct.h"
8 #include "gt/intel_gt.h"
10 #define CT_ERROR(_ct, _fmt, ...) \
11 DRM_DEV_ERROR(ct_to_dev(_ct), "CT: " _fmt, ##__VA_ARGS__)
12 #ifdef CONFIG_DRM_I915_DEBUG_GUC
13 #define CT_DEBUG(_ct, _fmt, ...) \
14 DRM_DEV_DEBUG_DRIVER(ct_to_dev(_ct), "CT: " _fmt, ##__VA_ARGS__)
16 #define CT_DEBUG(...) do { } while (0)
20 struct list_head link;
27 struct ct_incoming_request {
28 struct list_head link;
32 enum { CTB_SEND = 0, CTB_RECV = 1 };
34 enum { CTB_OWNER_HOST = 0 };
36 static void ct_incoming_request_worker_func(struct work_struct *w);
39 * intel_guc_ct_init_early - Initialize CT state without requiring device access
40 * @ct: pointer to CT struct
42 void intel_guc_ct_init_early(struct intel_guc_ct *ct)
44 spin_lock_init(&ct->requests.lock);
45 INIT_LIST_HEAD(&ct->requests.pending);
46 INIT_LIST_HEAD(&ct->requests.incoming);
47 INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func);
50 static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct)
52 return container_of(ct, struct intel_guc, ct);
55 static inline struct intel_gt *ct_to_gt(struct intel_guc_ct *ct)
57 return guc_to_gt(ct_to_guc(ct));
60 static inline struct drm_i915_private *ct_to_i915(struct intel_guc_ct *ct)
62 return ct_to_gt(ct)->i915;
65 static inline struct device *ct_to_dev(struct intel_guc_ct *ct)
67 return ct_to_i915(ct)->drm.dev;
70 static inline const char *guc_ct_buffer_type_to_str(u32 type)
73 case INTEL_GUC_CT_BUFFER_TYPE_SEND:
75 case INTEL_GUC_CT_BUFFER_TYPE_RECV:
82 static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc,
83 u32 cmds_addr, u32 size)
85 memset(desc, 0, sizeof(*desc));
86 desc->addr = cmds_addr;
88 desc->owner = CTB_OWNER_HOST;
91 static void guc_ct_buffer_desc_reset(struct guc_ct_buffer_desc *desc)
95 desc->is_in_error = 0;
98 static int guc_action_register_ct_buffer(struct intel_guc *guc,
103 INTEL_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER,
105 sizeof(struct guc_ct_buffer_desc),
109 /* Can't use generic send(), CT registration must go over MMIO */
110 return intel_guc_send_mmio(guc, action, ARRAY_SIZE(action), NULL, 0);
113 static int ct_register_buffer(struct intel_guc_ct *ct, u32 desc_addr, u32 type)
115 int err = guc_action_register_ct_buffer(ct_to_guc(ct), desc_addr, type);
118 CT_ERROR(ct, "Failed to register %s buffer (err=%d)\n",
119 guc_ct_buffer_type_to_str(type), err);
123 static int guc_action_deregister_ct_buffer(struct intel_guc *guc, u32 type)
126 INTEL_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER,
131 /* Can't use generic send(), CT deregistration must go over MMIO */
132 return intel_guc_send_mmio(guc, action, ARRAY_SIZE(action), NULL, 0);
135 static int ct_deregister_buffer(struct intel_guc_ct *ct, u32 type)
137 int err = guc_action_deregister_ct_buffer(ct_to_guc(ct), type);
140 CT_ERROR(ct, "Failed to deregister %s buffer (err=%d)\n",
141 guc_ct_buffer_type_to_str(type), err);
146 * intel_guc_ct_init - Init buffer-based communication
147 * @ct: pointer to CT struct
149 * Allocate memory required for buffer-based communication.
151 * Return: 0 on success, a negative errno code on failure.
153 int intel_guc_ct_init(struct intel_guc_ct *ct)
155 struct intel_guc *guc = ct_to_guc(ct);
162 /* We allocate 1 page to hold both descriptors and both buffers.
163 * ___________.....................
165 * |___________| PAGE/4
166 * :___________....................:
168 * |___________| PAGE/4
169 * :_______________________________:
172 * |_______________________________|
175 * |_______________________________|
177 * Each message can use a maximum of 32 dwords and we don't expect to
178 * have more than 1 in flight at any time, so we have enough space.
179 * Some logic further ahead will rely on the fact that there is only 1
180 * page and that it is always mapped, so if the size is changed the
181 * other code will need updating as well.
184 err = intel_guc_allocate_and_map_vma(guc, PAGE_SIZE, &ct->vma, &blob);
186 CT_ERROR(ct, "Failed to allocate CT channel (err=%d)\n", err);
190 CT_DEBUG(ct, "vma base=%#x\n", intel_guc_ggtt_offset(guc, ct->vma));
192 /* store pointers to desc and cmds */
193 for (i = 0; i < ARRAY_SIZE(ct->ctbs); i++) {
194 GEM_BUG_ON((i != CTB_SEND) && (i != CTB_RECV));
195 ct->ctbs[i].desc = blob + PAGE_SIZE/4 * i;
196 ct->ctbs[i].cmds = blob + PAGE_SIZE/4 * i + PAGE_SIZE/2;
203 * intel_guc_ct_fini - Fini buffer-based communication
204 * @ct: pointer to CT struct
206 * Deallocate memory required for buffer-based communication.
208 void intel_guc_ct_fini(struct intel_guc_ct *ct)
210 GEM_BUG_ON(ct->enabled);
212 i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP);
216 * intel_guc_ct_enable - Enable buffer based command transport.
217 * @ct: pointer to CT struct
219 * Return: 0 on success, a negative errno code on failure.
221 int intel_guc_ct_enable(struct intel_guc_ct *ct)
223 struct intel_guc *guc = ct_to_guc(ct);
224 u32 base, cmds, size;
228 GEM_BUG_ON(ct->enabled);
230 /* vma should be already allocated and map'ed */
231 GEM_BUG_ON(!ct->vma);
232 base = intel_guc_ggtt_offset(guc, ct->vma);
234 /* (re)initialize descriptors
235 * cmds buffers are in the second half of the blob page
237 for (i = 0; i < ARRAY_SIZE(ct->ctbs); i++) {
238 GEM_BUG_ON((i != CTB_SEND) && (i != CTB_RECV));
239 cmds = base + PAGE_SIZE / 4 * i + PAGE_SIZE / 2;
240 size = PAGE_SIZE / 4;
241 CT_DEBUG(ct, "%d: addr=%#x size=%u\n", i, cmds, size);
242 guc_ct_buffer_desc_init(ct->ctbs[i].desc, cmds, size);
246 * Register both CT buffers starting with RECV buffer.
247 * Descriptors are in first half of the blob.
249 err = ct_register_buffer(ct, base + PAGE_SIZE / 4 * CTB_RECV,
250 INTEL_GUC_CT_BUFFER_TYPE_RECV);
254 err = ct_register_buffer(ct, base + PAGE_SIZE / 4 * CTB_SEND,
255 INTEL_GUC_CT_BUFFER_TYPE_SEND);
264 ct_deregister_buffer(ct, INTEL_GUC_CT_BUFFER_TYPE_RECV);
266 CT_ERROR(ct, "Failed to open open CT channel (err=%d)\n", err);
271 * intel_guc_ct_disable - Disable buffer based command transport.
272 * @ct: pointer to CT struct
274 void intel_guc_ct_disable(struct intel_guc_ct *ct)
276 struct intel_guc *guc = ct_to_guc(ct);
278 GEM_BUG_ON(!ct->enabled);
282 if (intel_guc_is_fw_running(guc)) {
283 ct_deregister_buffer(ct, INTEL_GUC_CT_BUFFER_TYPE_SEND);
284 ct_deregister_buffer(ct, INTEL_GUC_CT_BUFFER_TYPE_RECV);
288 static u32 ct_get_next_fence(struct intel_guc_ct *ct)
290 /* For now it's trivial */
291 return ++ct->requests.last_fence;
295 * DOC: CTB Host to GuC request
297 * Format of the CTB Host to GuC request message is as follows::
299 * +------------+---------+---------+---------+---------+
300 * | msg[0] | [1] | [2] | ... | [n-1] |
301 * +------------+---------+---------+---------+---------+
302 * | MESSAGE | MESSAGE PAYLOAD |
303 * + HEADER +---------+---------+---------+---------+
304 * | | 0 | 1 | ... | n |
305 * +============+=========+=========+=========+=========+
306 * | len >= 1 | FENCE | request specific data |
307 * +------+-----+---------+---------+---------+---------+
309 * ^-----------------len-------------------^
312 static int ct_write(struct intel_guc_ct *ct,
314 u32 len /* in dwords */,
318 struct intel_guc_ct_buffer *ctb = &ct->ctbs[CTB_SEND];
319 struct guc_ct_buffer_desc *desc = ctb->desc;
320 u32 head = desc->head;
321 u32 tail = desc->tail;
322 u32 size = desc->size;
325 u32 *cmds = ctb->cmds;
328 if (unlikely(desc->is_in_error))
331 if (unlikely(!IS_ALIGNED(head | tail | size, 4) ||
332 (tail | head) >= size))
335 /* later calculations will be done in dwords */
341 * tail == head condition indicates empty. GuC FW does not support
342 * using up the entire buffer to get tail == head meaning full.
345 used = (size - head) + tail;
349 /* make sure there is a space including extra dw for the fence */
350 if (unlikely(used + len + 1 >= size))
354 * Write the message. The format is the following:
355 * DW0: header (including action code)
359 header = (len << GUC_CT_MSG_LEN_SHIFT) |
360 (GUC_CT_MSG_WRITE_FENCE_TO_DESC) |
361 (want_response ? GUC_CT_MSG_SEND_STATUS : 0) |
362 (action[0] << GUC_CT_MSG_ACTION_SHIFT);
364 CT_DEBUG(ct, "writing %*ph %*ph %*ph\n",
365 4, &header, 4, &fence, 4 * (len - 1), &action[1]);
368 tail = (tail + 1) % size;
371 tail = (tail + 1) % size;
373 for (i = 1; i < len; i++) {
374 cmds[tail] = action[i];
375 tail = (tail + 1) % size;
377 GEM_BUG_ON(tail > size);
379 /* now update desc tail (back in bytes) */
380 desc->tail = tail * 4;
384 CT_ERROR(ct, "Corrupted descriptor addr=%#x head=%u tail=%u size=%u\n",
385 desc->addr, desc->head, desc->tail, desc->size);
386 desc->is_in_error = 1;
391 * wait_for_ctb_desc_update - Wait for the CT buffer descriptor update.
392 * @desc: buffer descriptor
393 * @fence: response fence
394 * @status: placeholder for status
396 * Guc will update CT buffer descriptor with new fence and status
397 * after processing the command identified by the fence. Wait for
398 * specified fence and then read from the descriptor status of the
402 * * 0 response received (status is valid)
403 * * -ETIMEDOUT no response within hardcoded timeout
404 * * -EPROTO no response, CT buffer is in error
406 static int wait_for_ctb_desc_update(struct guc_ct_buffer_desc *desc,
413 * Fast commands should complete in less than 10us, so sample quickly
414 * up to that length of time, then switch to a slower sleep-wait loop.
415 * No GuC command should ever take longer than 10ms.
417 #define done (READ_ONCE(desc->fence) == fence)
418 err = wait_for_us(done, 10);
420 err = wait_for(done, 10);
424 DRM_ERROR("CT: fence %u failed; reported fence=%u\n",
427 if (WARN_ON(desc->is_in_error)) {
428 /* Something went wrong with the messaging, try to reset
429 * the buffer and hope for the best
431 guc_ct_buffer_desc_reset(desc);
436 *status = desc->status;
441 * wait_for_ct_request_update - Wait for CT request state update.
442 * @req: pointer to pending request
443 * @status: placeholder for status
445 * For each sent request, Guc shall send bac CT response message.
446 * Our message handler will update status of tracked request once
447 * response message with given fence is received. Wait here and
448 * check for valid response status value.
451 * * 0 response received (status is valid)
452 * * -ETIMEDOUT no response within hardcoded timeout
454 static int wait_for_ct_request_update(struct ct_request *req, u32 *status)
459 * Fast commands should complete in less than 10us, so sample quickly
460 * up to that length of time, then switch to a slower sleep-wait loop.
461 * No GuC command should ever take longer than 10ms.
463 #define done INTEL_GUC_MSG_IS_RESPONSE(READ_ONCE(req->status))
464 err = wait_for_us(done, 10);
466 err = wait_for(done, 10);
470 DRM_ERROR("CT: fence %u err %d\n", req->fence, err);
472 *status = req->status;
476 static int ct_send(struct intel_guc_ct *ct,
480 u32 response_buf_size,
483 struct intel_guc_ct_buffer *ctb = &ct->ctbs[CTB_SEND];
484 struct guc_ct_buffer_desc *desc = ctb->desc;
485 struct ct_request request;
490 GEM_BUG_ON(!ct->enabled);
492 GEM_BUG_ON(len & ~GUC_CT_MSG_LEN_MASK);
493 GEM_BUG_ON(!response_buf && response_buf_size);
495 fence = ct_get_next_fence(ct);
496 request.fence = fence;
498 request.response_len = response_buf_size;
499 request.response_buf = response_buf;
501 spin_lock_irqsave(&ct->requests.lock, flags);
502 list_add_tail(&request.link, &ct->requests.pending);
503 spin_unlock_irqrestore(&ct->requests.lock, flags);
505 err = ct_write(ct, action, len, fence, !!response_buf);
509 intel_guc_notify(ct_to_guc(ct));
512 err = wait_for_ct_request_update(&request, status);
514 err = wait_for_ctb_desc_update(desc, fence, status);
518 if (!INTEL_GUC_MSG_IS_RESPONSE_SUCCESS(*status)) {
524 /* There shall be no data in the status */
525 WARN_ON(INTEL_GUC_MSG_TO_DATA(request.status));
526 /* Return actual response len */
527 err = request.response_len;
529 /* There shall be no response payload */
530 WARN_ON(request.response_len);
531 /* Return data decoded from the status dword */
532 err = INTEL_GUC_MSG_TO_DATA(*status);
536 spin_lock_irqsave(&ct->requests.lock, flags);
537 list_del(&request.link);
538 spin_unlock_irqrestore(&ct->requests.lock, flags);
544 * Command Transport (CT) buffer based GuC send function.
546 int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len,
547 u32 *response_buf, u32 response_buf_size)
549 struct intel_guc *guc = ct_to_guc(ct);
550 u32 status = ~0; /* undefined */
553 if (unlikely(!ct->enabled)) {
554 WARN(1, "Unexpected send: action=%#x\n", *action);
558 mutex_lock(&guc->send_mutex);
560 ret = ct_send(ct, action, len, response_buf, response_buf_size, &status);
561 if (unlikely(ret < 0)) {
562 CT_ERROR(ct, "Sending action %#x failed (err=%d status=%#X)\n",
563 action[0], ret, status);
564 } else if (unlikely(ret)) {
565 CT_DEBUG(ct, "send action %#x returned %d (%#x)\n",
566 action[0], ret, ret);
569 mutex_unlock(&guc->send_mutex);
573 static inline unsigned int ct_header_get_len(u32 header)
575 return (header >> GUC_CT_MSG_LEN_SHIFT) & GUC_CT_MSG_LEN_MASK;
578 static inline unsigned int ct_header_get_action(u32 header)
580 return (header >> GUC_CT_MSG_ACTION_SHIFT) & GUC_CT_MSG_ACTION_MASK;
583 static inline bool ct_header_is_response(u32 header)
585 return !!(header & GUC_CT_MSG_IS_RESPONSE);
588 static int ct_read(struct intel_guc_ct *ct, u32 *data)
590 struct intel_guc_ct_buffer *ctb = &ct->ctbs[CTB_RECV];
591 struct guc_ct_buffer_desc *desc = ctb->desc;
592 u32 head = desc->head;
593 u32 tail = desc->tail;
594 u32 size = desc->size;
595 u32 *cmds = ctb->cmds;
600 if (unlikely(desc->is_in_error))
603 if (unlikely(!IS_ALIGNED(head | tail | size, 4) ||
604 (tail | head) >= size))
607 /* later calculations will be done in dwords */
612 /* tail == head condition indicates empty */
613 available = tail - head;
614 if (unlikely(available == 0))
617 /* beware of buffer wrap case */
618 if (unlikely(available < 0))
620 CT_DEBUG(ct, "available %d (%u:%u)\n", available, head, tail);
621 GEM_BUG_ON(available < 0);
623 data[0] = cmds[head];
624 head = (head + 1) % size;
626 /* message len with header */
627 len = ct_header_get_len(data[0]) + 1;
628 if (unlikely(len > (u32)available)) {
629 CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n",
631 4 * (head + available - 1 > size ?
632 size - head : available - 1), &cmds[head],
633 4 * (head + available - 1 > size ?
634 available - 1 - size + head : 0), &cmds[0]);
638 for (i = 1; i < len; i++) {
639 data[i] = cmds[head];
640 head = (head + 1) % size;
642 CT_DEBUG(ct, "received %*ph\n", 4 * len, data);
644 desc->head = head * 4;
648 CT_ERROR(ct, "Corrupted descriptor addr=%#x head=%u tail=%u size=%u\n",
649 desc->addr, desc->head, desc->tail, desc->size);
650 desc->is_in_error = 1;
655 * DOC: CTB GuC to Host response
657 * Format of the CTB GuC to Host response message is as follows::
659 * +------------+---------+---------+---------+---------+---------+
660 * | msg[0] | [1] | [2] | [3] | ... | [n-1] |
661 * +------------+---------+---------+---------+---------+---------+
662 * | MESSAGE | MESSAGE PAYLOAD |
663 * + HEADER +---------+---------+---------+---------+---------+
664 * | | 0 | 1 | 2 | ... | n |
665 * +============+=========+=========+=========+=========+=========+
666 * | len >= 2 | FENCE | STATUS | response specific data |
667 * +------+-----+---------+---------+---------+---------+---------+
669 * ^-----------------------len-----------------------^
672 static int ct_handle_response(struct intel_guc_ct *ct, const u32 *msg)
675 u32 len = ct_header_get_len(header);
676 u32 msgsize = (len + 1) * sizeof(u32); /* msg size in bytes w/header */
680 struct ct_request *req;
683 GEM_BUG_ON(!ct_header_is_response(header));
684 GEM_BUG_ON(!in_irq());
686 /* Response payload shall at least include fence and status */
687 if (unlikely(len < 2)) {
688 CT_ERROR(ct, "Corrupted response %*ph\n", msgsize, msg);
696 /* Format of the status follows RESPONSE message */
697 if (unlikely(!INTEL_GUC_MSG_IS_RESPONSE(status))) {
698 CT_ERROR(ct, "Corrupted response %*ph\n", msgsize, msg);
702 CT_DEBUG(ct, "response fence %u status %#x\n", fence, status);
704 spin_lock(&ct->requests.lock);
705 list_for_each_entry(req, &ct->requests.pending, link) {
706 if (unlikely(fence != req->fence)) {
707 CT_DEBUG(ct, "request %u awaits response\n",
711 if (unlikely(datalen > req->response_len)) {
712 CT_ERROR(ct, "Response for %u is too long %*ph\n",
713 req->fence, msgsize, msg);
717 memcpy(req->response_buf, msg + 3, 4 * datalen);
718 req->response_len = datalen;
719 WRITE_ONCE(req->status, status);
723 spin_unlock(&ct->requests.lock);
726 CT_ERROR(ct, "Unsolicited response %*ph\n", msgsize, msg);
730 static void ct_process_request(struct intel_guc_ct *ct,
731 u32 action, u32 len, const u32 *payload)
733 struct intel_guc *guc = ct_to_guc(ct);
736 CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload);
739 case INTEL_GUC_ACTION_DEFAULT:
740 ret = intel_guc_to_host_process_recv_msg(guc, payload, len);
742 goto fail_unexpected;
747 CT_ERROR(ct, "Unexpected request %x %*ph\n",
748 action, 4 * len, payload);
753 static bool ct_process_incoming_requests(struct intel_guc_ct *ct)
756 struct ct_incoming_request *request;
761 spin_lock_irqsave(&ct->requests.lock, flags);
762 request = list_first_entry_or_null(&ct->requests.incoming,
763 struct ct_incoming_request, link);
765 list_del(&request->link);
766 done = !!list_empty(&ct->requests.incoming);
767 spin_unlock_irqrestore(&ct->requests.lock, flags);
772 header = request->msg[0];
773 payload = &request->msg[1];
774 ct_process_request(ct,
775 ct_header_get_action(header),
776 ct_header_get_len(header),
783 static void ct_incoming_request_worker_func(struct work_struct *w)
785 struct intel_guc_ct *ct =
786 container_of(w, struct intel_guc_ct, requests.worker);
789 done = ct_process_incoming_requests(ct);
791 queue_work(system_unbound_wq, &ct->requests.worker);
795 * DOC: CTB GuC to Host request
797 * Format of the CTB GuC to Host request message is as follows::
799 * +------------+---------+---------+---------+---------+---------+
800 * | msg[0] | [1] | [2] | [3] | ... | [n-1] |
801 * +------------+---------+---------+---------+---------+---------+
802 * | MESSAGE | MESSAGE PAYLOAD |
803 * + HEADER +---------+---------+---------+---------+---------+
804 * | | 0 | 1 | 2 | ... | n |
805 * +============+=========+=========+=========+=========+=========+
806 * | len | request specific data |
807 * +------+-----+---------+---------+---------+---------+---------+
809 * ^-----------------------len-----------------------^
812 static int ct_handle_request(struct intel_guc_ct *ct, const u32 *msg)
815 u32 len = ct_header_get_len(header);
816 u32 msgsize = (len + 1) * sizeof(u32); /* msg size in bytes w/header */
817 struct ct_incoming_request *request;
820 GEM_BUG_ON(ct_header_is_response(header));
822 request = kmalloc(sizeof(*request) + msgsize, GFP_ATOMIC);
823 if (unlikely(!request)) {
824 CT_ERROR(ct, "Dropping request %*ph\n", msgsize, msg);
825 return 0; /* XXX: -ENOMEM ? */
827 memcpy(request->msg, msg, msgsize);
829 spin_lock_irqsave(&ct->requests.lock, flags);
830 list_add_tail(&request->link, &ct->requests.incoming);
831 spin_unlock_irqrestore(&ct->requests.lock, flags);
833 queue_work(system_unbound_wq, &ct->requests.worker);
838 * When we're communicating with the GuC over CT, GuC uses events
839 * to notify us about new messages being posted on the RECV buffer.
841 void intel_guc_ct_event_handler(struct intel_guc_ct *ct)
843 u32 msg[GUC_CT_MSG_LEN_MASK + 1]; /* one extra dw for the header */
846 if (unlikely(!ct->enabled)) {
847 WARN(1, "Unexpected GuC event received while CT disabled!\n");
852 err = ct_read(ct, msg);
856 if (ct_header_is_response(msg[0]))
857 err = ct_handle_response(ct, msg);
859 err = ct_handle_request(ct, msg);