1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
3 // This file is provided under a dual BSD/GPLv2 license. When using or
4 // redistributing this file, you may do so under either license.
6 // Copyright(c) 2022 Intel Corporation. All rights reserved.
8 // Authors: Rander Wang <rander.wang@linux.intel.com>
9 // Peter Ujfalusi <peter.ujfalusi@linux.intel.com>
11 #include <sound/sof/header.h>
12 #include <sound/sof/ipc4/header.h>
14 #include "sof-audio.h"
15 #include "ipc4-priv.h"
19 #define sof_ipc4_dump_payload(sdev, ipc_data, size) \
20 print_hex_dump_debug("Message payload: ", \
22 16, 4, ipc_data, size, false)
24 #define sof_ipc4_dump_payload(sdev, ipc_data, size) do { } while (0)
27 static const struct sof_ipc4_fw_status {
31 {0, "The operation was successful"},
32 {1, "Invalid parameter specified"},
33 {2, "Unknown message type specified"},
34 {3, "Not enough space in the IPC reply buffer to complete the request"},
35 {4, "The system or resource is busy"},
36 {5, "Replaced ADSP IPC PENDING (unused)"},
37 {6, "Unknown error while processing the request"},
38 {7, "Unsupported operation requested"},
39 {8, "Reserved (ADSP_STAGE_UNINITIALIZED removed)"},
40 {9, "Specified resource not found"},
41 {10, "A resource's ID requested to be created is already assigned"},
42 {11, "Reserved (ADSP_IPC_OUT_OF_MIPS removed)"},
43 {12, "Required resource is in invalid state"},
44 {13, "Requested power transition failed to complete"},
45 {14, "Manifest of the library being loaded is invalid"},
46 {15, "Requested service or data is unavailable on the target platform"},
47 {42, "Library target address is out of storage memory range"},
49 {44, "Image verification by CSE failed"},
50 {100, "General module management error"},
51 {101, "Module loading failed"},
52 {102, "Integrity check of the loaded module content failed"},
53 {103, "Attempt to unload code of the module in use"},
54 {104, "Other failure of module instance initialization request"},
55 {105, "Reserved (ADSP_IPC_OUT_OF_MIPS removed)"},
56 {106, "Reserved (ADSP_IPC_CONFIG_GET_ERROR removed)"},
57 {107, "Reserved (ADSP_IPC_CONFIG_SET_ERROR removed)"},
58 {108, "Reserved (ADSP_IPC_LARGE_CONFIG_GET_ERROR removed)"},
59 {109, "Reserved (ADSP_IPC_LARGE_CONFIG_SET_ERROR removed)"},
60 {110, "Invalid (out of range) module ID provided"},
61 {111, "Invalid module instance ID provided"},
62 {112, "Invalid queue (pin) ID provided"},
63 {113, "Invalid destination queue (pin) ID provided"},
64 {114, "Reserved (ADSP_IPC_BIND_UNBIND_DST_SINK_UNSUPPORTED removed)"},
65 {115, "Reserved (ADSP_IPC_UNLOAD_INST_EXISTS removed)"},
66 {116, "Invalid target code ID provided"},
67 {117, "Injection DMA buffer is too small for probing the input pin"},
68 {118, "Extraction DMA buffer is too small for probing the output pin"},
69 {120, "Invalid ID of configuration item provided in TLV list"},
70 {121, "Invalid length of configuration item provided in TLV list"},
71 {122, "Invalid structure of configuration item provided"},
72 {140, "Initialization of DMA Gateway failed"},
73 {141, "Invalid ID of gateway provided"},
74 {142, "Setting state of DMA Gateway failed"},
75 {143, "DMA_CONTROL message targeting gateway not allocated yet"},
76 {150, "Attempt to configure SCLK while I2S port is running"},
77 {151, "Attempt to configure MCLK while I2S port is running"},
78 {152, "Attempt to stop SCLK that is not running"},
79 {153, "Attempt to stop MCLK that is not running"},
80 {160, "Reserved (ADSP_IPC_PIPELINE_NOT_INITIALIZED removed)"},
81 {161, "Reserved (ADSP_IPC_PIPELINE_NOT_EXIST removed)"},
82 {162, "Reserved (ADSP_IPC_PIPELINE_SAVE_FAILED removed)"},
83 {163, "Reserved (ADSP_IPC_PIPELINE_RESTORE_FAILED removed)"},
84 {165, "Reserved (ADSP_IPC_PIPELINE_ALREADY_EXISTS removed)"},
87 static int sof_ipc4_check_reply_status(struct snd_sof_dev *sdev, u32 status)
91 status &= SOF_IPC4_REPLY_STATUS;
96 for (i = 0; i < ARRAY_SIZE(ipc4_status); i++) {
97 if (ipc4_status[i].status == status) {
98 dev_err(sdev->dev, "FW reported error: %u - %s\n",
99 status, ipc4_status[i].msg);
104 if (i == ARRAY_SIZE(ipc4_status))
105 dev_err(sdev->dev, "FW reported error: %u - Unknown\n", status);
130 #if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC)
131 #define DBG_IPC4_MSG_TYPE_ENTRY(type) [SOF_IPC4_##type] = #type
132 static const char * const ipc4_dbg_mod_msg_type[] = {
133 DBG_IPC4_MSG_TYPE_ENTRY(MOD_INIT_INSTANCE),
134 DBG_IPC4_MSG_TYPE_ENTRY(MOD_CONFIG_GET),
135 DBG_IPC4_MSG_TYPE_ENTRY(MOD_CONFIG_SET),
136 DBG_IPC4_MSG_TYPE_ENTRY(MOD_LARGE_CONFIG_GET),
137 DBG_IPC4_MSG_TYPE_ENTRY(MOD_LARGE_CONFIG_SET),
138 DBG_IPC4_MSG_TYPE_ENTRY(MOD_BIND),
139 DBG_IPC4_MSG_TYPE_ENTRY(MOD_UNBIND),
140 DBG_IPC4_MSG_TYPE_ENTRY(MOD_SET_DX),
141 DBG_IPC4_MSG_TYPE_ENTRY(MOD_SET_D0IX),
142 DBG_IPC4_MSG_TYPE_ENTRY(MOD_ENTER_MODULE_RESTORE),
143 DBG_IPC4_MSG_TYPE_ENTRY(MOD_EXIT_MODULE_RESTORE),
144 DBG_IPC4_MSG_TYPE_ENTRY(MOD_DELETE_INSTANCE),
147 static const char * const ipc4_dbg_glb_msg_type[] = {
148 DBG_IPC4_MSG_TYPE_ENTRY(GLB_BOOT_CONFIG),
149 DBG_IPC4_MSG_TYPE_ENTRY(GLB_ROM_CONTROL),
150 DBG_IPC4_MSG_TYPE_ENTRY(GLB_IPCGATEWAY_CMD),
151 DBG_IPC4_MSG_TYPE_ENTRY(GLB_PERF_MEASUREMENTS_CMD),
152 DBG_IPC4_MSG_TYPE_ENTRY(GLB_CHAIN_DMA),
153 DBG_IPC4_MSG_TYPE_ENTRY(GLB_LOAD_MULTIPLE_MODULES),
154 DBG_IPC4_MSG_TYPE_ENTRY(GLB_UNLOAD_MULTIPLE_MODULES),
155 DBG_IPC4_MSG_TYPE_ENTRY(GLB_CREATE_PIPELINE),
156 DBG_IPC4_MSG_TYPE_ENTRY(GLB_DELETE_PIPELINE),
157 DBG_IPC4_MSG_TYPE_ENTRY(GLB_SET_PIPELINE_STATE),
158 DBG_IPC4_MSG_TYPE_ENTRY(GLB_GET_PIPELINE_STATE),
159 DBG_IPC4_MSG_TYPE_ENTRY(GLB_GET_PIPELINE_CONTEXT_SIZE),
160 DBG_IPC4_MSG_TYPE_ENTRY(GLB_SAVE_PIPELINE),
161 DBG_IPC4_MSG_TYPE_ENTRY(GLB_RESTORE_PIPELINE),
162 DBG_IPC4_MSG_TYPE_ENTRY(GLB_LOAD_LIBRARY),
163 DBG_IPC4_MSG_TYPE_ENTRY(GLB_INTERNAL_MESSAGE),
164 DBG_IPC4_MSG_TYPE_ENTRY(GLB_NOTIFICATION),
167 #define DBG_IPC4_NOTIFICATION_TYPE_ENTRY(type) [SOF_IPC4_NOTIFY_##type] = #type
168 static const char * const ipc4_dbg_notification_type[] = {
169 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(PHRASE_DETECTED),
170 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(RESOURCE_EVENT),
171 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(LOG_BUFFER_STATUS),
172 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(TIMESTAMP_CAPTURED),
173 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(FW_READY),
174 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(FW_AUD_CLASS_RESULT),
175 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(EXCEPTION_CAUGHT),
176 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(MODULE_NOTIFICATION),
177 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(PROBE_DATA_AVAILABLE),
178 DBG_IPC4_NOTIFICATION_TYPE_ENTRY(ASYNC_MSG_SRVC_MESSAGE),
181 static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_msg *msg,
182 bool data_size_valid)
185 const u8 *str2 = NULL;
186 const u8 *str = NULL;
188 val = msg->primary & SOF_IPC4_MSG_TARGET_MASK;
189 type = SOF_IPC4_MSG_TYPE_GET(msg->primary);
191 if (val == SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG)) {
193 if (type < SOF_IPC4_MOD_TYPE_LAST)
194 str = ipc4_dbg_mod_msg_type[type];
196 str = "Unknown Module message type";
198 /* Global FW message */
199 if (type < SOF_IPC4_GLB_TYPE_LAST)
200 str = ipc4_dbg_glb_msg_type[type];
202 str = "Unknown Global message type";
204 if (type == SOF_IPC4_GLB_NOTIFICATION) {
205 /* Notification message */
206 u32 notif = SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary);
208 if (notif < SOF_IPC4_NOTIFY_TYPE_LAST)
209 str2 = ipc4_dbg_notification_type[notif];
211 str2 = "Unknown Global notification";
216 if (data_size_valid && msg->data_size)
217 dev_dbg(dev, "%s: %#x|%#x: %s|%s [data size: %zu]\n",
218 text, msg->primary, msg->extension, str, str2,
221 dev_dbg(dev, "%s: %#x|%#x: %s|%s\n", text, msg->primary,
222 msg->extension, str, str2);
224 if (data_size_valid && msg->data_size)
225 dev_dbg(dev, "%s: %#x|%#x: %s [data size: %zu]\n",
226 text, msg->primary, msg->extension, str,
229 dev_dbg(dev, "%s: %#x|%#x: %s\n", text, msg->primary,
230 msg->extension, str);
233 #else /* CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC */
234 static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_msg *msg,
235 bool data_size_valid)
237 if (data_size_valid && msg->data_size)
238 dev_dbg(dev, "%s: %#x|%#x [data size: %zu]\n", text,
239 msg->primary, msg->extension, msg->data_size);
241 dev_dbg(dev, "%s: %#x|%#x\n", text, msg->primary, msg->extension);
245 static int sof_ipc4_get_reply(struct snd_sof_dev *sdev)
247 struct snd_sof_ipc_msg *msg = sdev->msg;
248 struct sof_ipc4_msg *ipc4_reply;
251 /* get the generic reply */
252 ipc4_reply = msg->reply_data;
254 sof_ipc4_log_header(sdev->dev, "ipc tx reply", ipc4_reply, false);
256 ret = sof_ipc4_check_reply_status(sdev, ipc4_reply->primary);
260 /* No other information is expected for non large config get replies */
261 if (!msg->reply_size || !SOF_IPC4_MSG_IS_MODULE_MSG(ipc4_reply->primary) ||
262 (SOF_IPC4_MSG_TYPE_GET(ipc4_reply->primary) != SOF_IPC4_MOD_LARGE_CONFIG_GET))
265 /* Read the requested payload */
266 snd_sof_dsp_mailbox_read(sdev, sdev->dsp_box.offset, ipc4_reply->data_ptr,
272 /* wait for IPC message reply */
273 static int ipc4_wait_tx_done(struct snd_sof_ipc *ipc, void *reply_data)
275 struct snd_sof_ipc_msg *msg = &ipc->msg;
276 struct sof_ipc4_msg *ipc4_msg = msg->msg_data;
277 struct snd_sof_dev *sdev = ipc->sdev;
280 /* wait for DSP IPC completion */
281 ret = wait_event_timeout(msg->waitq, msg->ipc_complete,
282 msecs_to_jiffies(sdev->ipc_timeout));
284 dev_err(sdev->dev, "ipc timed out for %#x|%#x\n",
285 ipc4_msg->primary, ipc4_msg->extension);
289 if (msg->reply_error) {
290 dev_err(sdev->dev, "ipc error for msg %#x|%#x\n",
291 ipc4_msg->primary, ipc4_msg->extension);
292 ret = msg->reply_error;
295 struct sof_ipc4_msg *ipc4_reply = msg->reply_data;
296 struct sof_ipc4_msg *ipc4_reply_data = reply_data;
298 /* Copy the header */
299 ipc4_reply_data->header_u64 = ipc4_reply->header_u64;
300 if (msg->reply_size && ipc4_reply_data->data_ptr) {
301 /* copy the payload returned from DSP */
302 memcpy(ipc4_reply_data->data_ptr, ipc4_reply->data_ptr,
304 ipc4_reply_data->data_size = msg->reply_size;
309 sof_ipc4_log_header(sdev->dev, "ipc tx done ", ipc4_msg, true);
312 /* re-enable dumps after successful IPC tx */
313 if (sdev->ipc_dump_printed) {
314 sdev->dbg_dump_printed = false;
315 sdev->ipc_dump_printed = false;
321 static int ipc4_tx_msg_unlocked(struct snd_sof_ipc *ipc,
322 void *msg_data, size_t msg_bytes,
323 void *reply_data, size_t reply_bytes)
325 struct sof_ipc4_msg *ipc4_msg = msg_data;
326 struct snd_sof_dev *sdev = ipc->sdev;
329 if (msg_bytes > ipc->max_payload_size || reply_bytes > ipc->max_payload_size)
332 ret = sof_ipc_send_msg(sdev, msg_data, msg_bytes, reply_bytes);
334 dev_err_ratelimited(sdev->dev,
335 "%s: ipc message send for %#x|%#x failed: %d\n",
336 __func__, ipc4_msg->primary, ipc4_msg->extension, ret);
340 sof_ipc4_log_header(sdev->dev, "ipc tx ", msg_data, true);
342 /* now wait for completion */
343 return ipc4_wait_tx_done(ipc, reply_data);
346 static int sof_ipc4_tx_msg(struct snd_sof_dev *sdev, void *msg_data, size_t msg_bytes,
347 void *reply_data, size_t reply_bytes, bool no_pm)
349 struct snd_sof_ipc *ipc = sdev->ipc;
351 struct sof_ipc4_msg *msg = NULL;
358 /* Serialise IPC TX */
359 mutex_lock(&ipc->tx_mutex);
361 ret = ipc4_tx_msg_unlocked(ipc, msg_data, msg_bytes, reply_data, reply_bytes);
363 mutex_unlock(&ipc->tx_mutex);
366 /* payload is indicated by non zero msg/reply_bytes */
369 else if (reply_bytes)
373 sof_ipc4_dump_payload(sdev, msg->data_ptr, msg->data_size);
379 static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data,
380 size_t payload_bytes, bool set)
382 size_t payload_limit = sdev->ipc->max_payload_size;
383 struct sof_ipc4_msg *ipc4_msg = data;
384 struct sof_ipc4_msg tx = {{ 0 }};
385 struct sof_ipc4_msg rx = {{ 0 }};
386 size_t remaining = payload_bytes;
394 if ((ipc4_msg->primary & SOF_IPC4_MSG_TARGET_MASK) !=
395 SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG))
398 ipc4_msg->primary &= ~SOF_IPC4_MSG_TYPE_MASK;
399 tx.primary = ipc4_msg->primary;
400 tx.extension = ipc4_msg->extension;
403 tx.primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_SET);
405 tx.primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_GET);
407 tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_SIZE_MASK;
408 tx.extension |= SOF_IPC4_MOD_EXT_MSG_SIZE(payload_bytes);
410 tx.extension |= SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK(1);
412 /* Serialise IPC TX */
413 mutex_lock(&sdev->ipc->tx_mutex);
416 size_t tx_size, rx_size;
418 if (remaining > payload_limit) {
419 chunk_size = payload_limit;
421 chunk_size = remaining;
423 tx.extension |= SOF_IPC4_MOD_EXT_MSG_LAST_BLOCK(1);
427 tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK_MASK;
428 tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_SIZE_MASK;
429 tx.extension |= SOF_IPC4_MOD_EXT_MSG_SIZE(offset);
433 tx.data_size = chunk_size;
434 tx.data_ptr = ipc4_msg->data_ptr + offset;
436 tx_size = chunk_size;
441 rx.data_size = chunk_size;
442 rx.data_ptr = ipc4_msg->data_ptr + offset;
445 rx_size = chunk_size;
448 /* Send the message for the current chunk */
449 ret = ipc4_tx_msg_unlocked(sdev->ipc, &tx, tx_size, &rx, rx_size);
452 "%s: large config %s failed at offset %zu: %d\n",
453 __func__, set ? "set" : "get", offset, ret);
457 if (!set && rx.extension & SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK_MASK) {
458 /* Verify the firmware reported total payload size */
459 rx_size = rx.extension & SOF_IPC4_MOD_EXT_MSG_SIZE_MASK;
461 if (rx_size > payload_bytes) {
463 "%s: Receive buffer (%zu) is too small for %zu\n",
464 __func__, payload_bytes, rx_size);
469 if (rx_size < chunk_size) {
470 chunk_size = rx_size;
472 } else if (rx_size < payload_bytes) {
477 offset += chunk_size;
478 remaining -= chunk_size;
481 /* Adjust the received data size if needed */
482 if (!set && payload_bytes != offset)
483 ipc4_msg->data_size = offset;
485 sof_ipc4_dump_payload(sdev, ipc4_msg->data_ptr, ipc4_msg->data_size);
488 mutex_unlock(&sdev->ipc->tx_mutex);
493 static int sof_ipc4_init_msg_memory(struct snd_sof_dev *sdev)
495 struct sof_ipc4_msg *ipc4_msg;
496 struct snd_sof_ipc_msg *msg = &sdev->ipc->msg;
498 /* TODO: get max_payload_size from firmware */
499 sdev->ipc->max_payload_size = SOF_IPC4_MSG_MAX_SIZE;
501 /* Allocate memory for the ipc4 container and the maximum payload */
502 msg->reply_data = devm_kzalloc(sdev->dev, sdev->ipc->max_payload_size +
503 sizeof(struct sof_ipc4_msg), GFP_KERNEL);
504 if (!msg->reply_data)
507 ipc4_msg = msg->reply_data;
508 ipc4_msg->data_ptr = msg->reply_data + sizeof(struct sof_ipc4_msg);
513 static int ipc4_fw_ready(struct snd_sof_dev *sdev, struct sof_ipc4_msg *ipc4_msg)
515 int inbox_offset, inbox_size, outbox_offset, outbox_size;
517 /* no need to re-check version/ABI for subsequent boots */
518 if (!sdev->first_boot)
521 /* Set up the windows for IPC communication */
522 inbox_offset = snd_sof_dsp_get_mailbox_offset(sdev);
523 if (inbox_offset < 0) {
524 dev_err(sdev->dev, "%s: No mailbox offset\n", __func__);
527 inbox_size = SOF_IPC4_MSG_MAX_SIZE;
528 outbox_offset = snd_sof_dsp_get_window_offset(sdev, 1);
529 outbox_size = SOF_IPC4_MSG_MAX_SIZE;
531 sdev->dsp_box.offset = inbox_offset;
532 sdev->dsp_box.size = inbox_size;
533 sdev->host_box.offset = outbox_offset;
534 sdev->host_box.size = outbox_size;
536 dev_dbg(sdev->dev, "mailbox upstream 0x%x - size 0x%x\n",
537 inbox_offset, inbox_size);
538 dev_dbg(sdev->dev, "mailbox downstream 0x%x - size 0x%x\n",
539 outbox_offset, outbox_size);
541 return sof_ipc4_init_msg_memory(sdev);
544 static void sof_ipc4_rx_msg(struct snd_sof_dev *sdev)
546 struct sof_ipc4_msg *ipc4_msg = sdev->ipc->msg.rx_data;
547 size_t data_size = 0;
550 if (!ipc4_msg || !SOF_IPC4_MSG_IS_NOTIFICATION(ipc4_msg->primary))
553 ipc4_msg->data_ptr = NULL;
554 ipc4_msg->data_size = 0;
556 sof_ipc4_log_header(sdev->dev, "ipc rx ", ipc4_msg, false);
558 switch (SOF_IPC4_NOTIFICATION_TYPE_GET(ipc4_msg->primary)) {
559 case SOF_IPC4_NOTIFY_FW_READY:
560 /* check for FW boot completion */
561 if (sdev->fw_state == SOF_FW_BOOT_IN_PROGRESS) {
562 err = ipc4_fw_ready(sdev, ipc4_msg);
564 sof_set_fw_state(sdev, SOF_FW_BOOT_READY_FAILED);
566 sof_set_fw_state(sdev, SOF_FW_BOOT_READY_OK);
568 /* wake up firmware loader */
569 wake_up(&sdev->boot_wait);
573 case SOF_IPC4_NOTIFY_RESOURCE_EVENT:
574 data_size = sizeof(struct sof_ipc4_notify_resource_data);
577 dev_dbg(sdev->dev, "%s: Unhandled DSP message: %#x|%#x\n", __func__,
578 ipc4_msg->primary, ipc4_msg->extension);
583 ipc4_msg->data_ptr = kmalloc(data_size, GFP_KERNEL);
584 if (!ipc4_msg->data_ptr)
587 ipc4_msg->data_size = data_size;
588 snd_sof_ipc_msg_data(sdev, NULL, ipc4_msg->data_ptr, ipc4_msg->data_size);
591 sof_ipc4_log_header(sdev->dev, "ipc rx done ", ipc4_msg, true);
594 kfree(ipc4_msg->data_ptr);
595 ipc4_msg->data_ptr = NULL;
596 ipc4_msg->data_size = 0;
600 const struct sof_ipc_ops ipc4_ops = {
601 .tx_msg = sof_ipc4_tx_msg,
602 .rx_msg = sof_ipc4_rx_msg,
603 .set_get_data = sof_ipc4_set_get_data,
604 .get_reply = sof_ipc4_get_reply,
605 .fw_loader = &ipc4_loader_ops,