sound/soc/sof/ipc4.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2 //
   3 // This file is provided under a dual BSD/GPLv2 license.  When using or
   4 // redistributing this file, you may do so under either license.
   5 //
   6 // Copyright(c) 2022 Intel Corporation. All rights reserved.
   7 //
   8 // Authors: Rander Wang <rander.wang@linux.intel.com>
   9 //          Peter Ujfalusi <peter.ujfalusi@linux.intel.com>
  10 //
  11 #include <linux/firmware.h>
  12 #include <sound/sof/header.h>
  13 #include <sound/sof/ipc4/header.h>
  14 #include "sof-priv.h"
  15 #include "sof-audio.h"
  16 #include "ipc4-fw-reg.h"
  17 #include "ipc4-priv.h"
  18 #include "ipc4-telemetry.h"
  19 #include "ops.h"
  20
  21 static const struct sof_ipc4_fw_status {
  22         int status;
  23         char *msg;
  24 } ipc4_status[] = {
  25         {0, "The operation was successful"},
  26         {1, "Invalid parameter specified"},
  27         {2, "Unknown message type specified"},
  28         {3, "Not enough space in the IPC reply buffer to complete the request"},
  29         {4, "The system or resource is busy"},
  30         {5, "Replaced ADSP IPC PENDING (unused)"},
  31         {6, "Unknown error while processing the request"},
  32         {7, "Unsupported operation requested"},
  33         {8, "Reserved (ADSP_STAGE_UNINITIALIZED removed)"},
  34         {9, "Specified resource not found"},
  35         {10, "A resource's ID requested to be created is already assigned"},
  36         {11, "Reserved (ADSP_IPC_OUT_OF_MIPS removed)"},
  37         {12, "Required resource is in invalid state"},
  38         {13, "Requested power transition failed to complete"},
  39         {14, "Manifest of the library being loaded is invalid"},
  40         {15, "Requested service or data is unavailable on the target platform"},
  41         {42, "Library target address is out of storage memory range"},
  42         {43, "Reserved"},
  43         {44, "Image verification by CSE failed"},
  44         {100, "General module management error"},
  45         {101, "Module loading failed"},
  46         {102, "Integrity check of the loaded module content failed"},
  47         {103, "Attempt to unload code of the module in use"},
  48         {104, "Other failure of module instance initialization request"},
  49         {105, "Reserved (ADSP_IPC_OUT_OF_MIPS removed)"},
  50         {106, "Reserved (ADSP_IPC_CONFIG_GET_ERROR removed)"},
  51         {107, "Reserved (ADSP_IPC_CONFIG_SET_ERROR removed)"},
  52         {108, "Reserved (ADSP_IPC_LARGE_CONFIG_GET_ERROR removed)"},
  53         {109, "Reserved (ADSP_IPC_LARGE_CONFIG_SET_ERROR removed)"},
  54         {110, "Invalid (out of range) module ID provided"},
  55         {111, "Invalid module instance ID provided"},
  56         {112, "Invalid queue (pin) ID provided"},
  57         {113, "Invalid destination queue (pin) ID provided"},
  58         {114, "Reserved (ADSP_IPC_BIND_UNBIND_DST_SINK_UNSUPPORTED removed)"},
  59         {115, "Reserved (ADSP_IPC_UNLOAD_INST_EXISTS removed)"},
  60         {116, "Invalid target code ID provided"},
  61         {117, "Injection DMA buffer is too small for probing the input pin"},
  62         {118, "Extraction DMA buffer is too small for probing the output pin"},
  63         {120, "Invalid ID of configuration item provided in TLV list"},
  64         {121, "Invalid length of configuration item provided in TLV list"},
  65         {122, "Invalid structure of configuration item provided"},
  66         {140, "Initialization of DMA Gateway failed"},
  67         {141, "Invalid ID of gateway provided"},
  68         {142, "Setting state of DMA Gateway failed"},
  69         {143, "DMA_CONTROL message targeting gateway not allocated yet"},
  70         {150, "Attempt to configure SCLK while I2S port is running"},
  71         {151, "Attempt to configure MCLK while I2S port is running"},
  72         {152, "Attempt to stop SCLK that is not running"},
  73         {153, "Attempt to stop MCLK that is not running"},
  74         {160, "Reserved (ADSP_IPC_PIPELINE_NOT_INITIALIZED removed)"},
  75         {161, "Reserved (ADSP_IPC_PIPELINE_NOT_EXIST removed)"},
  76         {162, "Reserved (ADSP_IPC_PIPELINE_SAVE_FAILED removed)"},
  77         {163, "Reserved (ADSP_IPC_PIPELINE_RESTORE_FAILED removed)"},
  78         {165, "Reserved (ADSP_IPC_PIPELINE_ALREADY_EXISTS removed)"},
  79 };
  80
  81 static int sof_ipc4_check_reply_status(struct snd_sof_dev *sdev, u32 status)
  82 {
  83         int i, ret;
  84
  85         status &= SOF_IPC4_REPLY_STATUS;
  86
  87         if (!status)
  88                 return 0;
  89
  90         for (i = 0; i < ARRAY_SIZE(ipc4_status); i++) {
  91                 if (ipc4_status[i].status == status) {
  92                         dev_err(sdev->dev, "FW reported error: %u - %s\n",
  93                                 status, ipc4_status[i].msg);
  94                         goto to_errno;
  95                 }
  96         }
  97
  98         if (i == ARRAY_SIZE(ipc4_status))
  99                 dev_err(sdev->dev, "FW reported error: %u - Unknown\n", status);
 100
 101 to_errno:
 102         switch (status) {
 103         case 2:
 104         case 15:
 105                 ret = -EOPNOTSUPP;
 106                 break;
 107         case 8:
 108         case 11:
 109         case 105 ... 109:
 110         case 114 ... 115:
 111         case 160 ... 163:
 112         case 165:
 113                 ret = -ENOENT;
 114                 break;
 115         case 4:
 116         case 150:
 117         case 151:
 118                 ret = -EBUSY;
 119                 break;
 120         default:
 121                 ret = -EINVAL;
 122                 break;
 123         }
 124
 125         return ret;
 126 }
 127
 128 #if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC)
 129 #define DBG_IPC4_MSG_TYPE_ENTRY(type)   [SOF_IPC4_##type] = #type
 130 static const char * const ipc4_dbg_mod_msg_type[] = {
 131         DBG_IPC4_MSG_TYPE_ENTRY(MOD_INIT_INSTANCE),
 132         DBG_IPC4_MSG_TYPE_ENTRY(MOD_CONFIG_GET),
 133         DBG_IPC4_MSG_TYPE_ENTRY(MOD_CONFIG_SET),
 134         DBG_IPC4_MSG_TYPE_ENTRY(MOD_LARGE_CONFIG_GET),
 135         DBG_IPC4_MSG_TYPE_ENTRY(MOD_LARGE_CONFIG_SET),
 136         DBG_IPC4_MSG_TYPE_ENTRY(MOD_BIND),
 137         DBG_IPC4_MSG_TYPE_ENTRY(MOD_UNBIND),
 138         DBG_IPC4_MSG_TYPE_ENTRY(MOD_SET_DX),
 139         DBG_IPC4_MSG_TYPE_ENTRY(MOD_SET_D0IX),
 140         DBG_IPC4_MSG_TYPE_ENTRY(MOD_ENTER_MODULE_RESTORE),
 141         DBG_IPC4_MSG_TYPE_ENTRY(MOD_EXIT_MODULE_RESTORE),
 142         DBG_IPC4_MSG_TYPE_ENTRY(MOD_DELETE_INSTANCE),
 143 };
 144
 145 static const char * const ipc4_dbg_glb_msg_type[] = {
 146         DBG_IPC4_MSG_TYPE_ENTRY(GLB_BOOT_CONFIG),
 147         DBG_IPC4_MSG_TYPE_ENTRY(GLB_ROM_CONTROL),
 148         DBG_IPC4_MSG_TYPE_ENTRY(GLB_IPCGATEWAY_CMD),
 149         DBG_IPC4_MSG_TYPE_ENTRY(GLB_PERF_MEASUREMENTS_CMD),
 150         DBG_IPC4_MSG_TYPE_ENTRY(GLB_CHAIN_DMA),
 151         DBG_IPC4_MSG_TYPE_ENTRY(GLB_LOAD_MULTIPLE_MODULES),
 152         DBG_IPC4_MSG_TYPE_ENTRY(GLB_UNLOAD_MULTIPLE_MODULES),
 153         DBG_IPC4_MSG_TYPE_ENTRY(GLB_CREATE_PIPELINE),
 154         DBG_IPC4_MSG_TYPE_ENTRY(GLB_DELETE_PIPELINE),
 155         DBG_IPC4_MSG_TYPE_ENTRY(GLB_SET_PIPELINE_STATE),
 156         DBG_IPC4_MSG_TYPE_ENTRY(GLB_GET_PIPELINE_STATE),
 157         DBG_IPC4_MSG_TYPE_ENTRY(GLB_GET_PIPELINE_CONTEXT_SIZE),
 158         DBG_IPC4_MSG_TYPE_ENTRY(GLB_SAVE_PIPELINE),
 159         DBG_IPC4_MSG_TYPE_ENTRY(GLB_RESTORE_PIPELINE),
 160         DBG_IPC4_MSG_TYPE_ENTRY(GLB_LOAD_LIBRARY),
 161         DBG_IPC4_MSG_TYPE_ENTRY(GLB_LOAD_LIBRARY_PREPARE),
 162         DBG_IPC4_MSG_TYPE_ENTRY(GLB_INTERNAL_MESSAGE),
 163         DBG_IPC4_MSG_TYPE_ENTRY(GLB_NOTIFICATION),
 164 };
 165
 166 #define DBG_IPC4_NOTIFICATION_TYPE_ENTRY(type)  [SOF_IPC4_NOTIFY_##type] = #type
 167 static const char * const ipc4_dbg_notification_type[] = {
 168         DBG_IPC4_NOTIFICATION_TYPE_ENTRY(PHRASE_DETECTED),
 169         DBG_IPC4_NOTIFICATION_TYPE_ENTRY(RESOURCE_EVENT),
 170         DBG_IPC4_NOTIFICATION_TYPE_ENTRY(LOG_BUFFER_STATUS),
 171         DBG_IPC4_NOTIFICATION_TYPE_ENTRY(TIMESTAMP_CAPTURED),
 172         DBG_IPC4_NOTIFICATION_TYPE_ENTRY(FW_READY),
 173         DBG_IPC4_NOTIFICATION_TYPE_ENTRY(FW_AUD_CLASS_RESULT),
 174         DBG_IPC4_NOTIFICATION_TYPE_ENTRY(EXCEPTION_CAUGHT),
 175         DBG_IPC4_NOTIFICATION_TYPE_ENTRY(MODULE_NOTIFICATION),
 176         DBG_IPC4_NOTIFICATION_TYPE_ENTRY(PROBE_DATA_AVAILABLE),
 177         DBG_IPC4_NOTIFICATION_TYPE_ENTRY(ASYNC_MSG_SRVC_MESSAGE),
 178 };
 179
 180 static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_msg *msg,
 181                                 bool data_size_valid)
 182 {
 183         u32 val, type;
 184         const u8 *str2 = NULL;
 185         const u8 *str = NULL;
 186
 187         val = msg->primary & SOF_IPC4_MSG_TARGET_MASK;
 188         type = SOF_IPC4_MSG_TYPE_GET(msg->primary);
 189
 190         if (val == SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG)) {
 191                 /* Module message */
 192                 if (type < SOF_IPC4_MOD_TYPE_LAST)
 193                         str = ipc4_dbg_mod_msg_type[type];
 194                 if (!str)
 195                         str = "Unknown Module message type";
 196         } else {
 197                 /* Global FW message */
 198                 if (type < SOF_IPC4_GLB_TYPE_LAST)
 199                         str = ipc4_dbg_glb_msg_type[type];
 200                 if (!str)
 201                         str = "Unknown Global message type";
 202
 203                 if (type == SOF_IPC4_GLB_NOTIFICATION) {
 204                         /* Notification message */
 205                         u32 notif = SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary);
 206
 207                         /* Do not print log buffer notification if not desired */
 208                         if (notif == SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS &&
 209                             !sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS))
 210                                 return;
 211
 212                         if (notif < SOF_IPC4_NOTIFY_TYPE_LAST)
 213                                 str2 = ipc4_dbg_notification_type[notif];
 214                         if (!str2)
 215                                 str2 = "Unknown Global notification";
 216                 }
 217         }
 218
 219         if (str2) {
 220                 if (data_size_valid && msg->data_size)
 221                         dev_dbg(dev, "%s: %#x|%#x: %s|%s [data size: %zu]\n",
 222                                 text, msg->primary, msg->extension, str, str2,
 223                                 msg->data_size);
 224                 else
 225                         dev_dbg(dev, "%s: %#x|%#x: %s|%s\n", text, msg->primary,
 226                                 msg->extension, str, str2);
 227         } else {
 228                 if (data_size_valid && msg->data_size)
 229                         dev_dbg(dev, "%s: %#x|%#x: %s [data size: %zu]\n",
 230                                 text, msg->primary, msg->extension, str,
 231                                 msg->data_size);
 232                 else
 233                         dev_dbg(dev, "%s: %#x|%#x: %s\n", text, msg->primary,
 234                                 msg->extension, str);
 235         }
 236 }
 237 #else /* CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC */
 238 static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_msg *msg,
 239                                 bool data_size_valid)
 240 {
 241         /* Do not print log buffer notification if not desired */
 242         if (!sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS) &&
 243             !SOF_IPC4_MSG_IS_MODULE_MSG(msg->primary) &&
 244             SOF_IPC4_MSG_TYPE_GET(msg->primary) == SOF_IPC4_GLB_NOTIFICATION &&
 245             SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary) == SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS)
 246                 return;
 247
 248         if (data_size_valid && msg->data_size)
 249                 dev_dbg(dev, "%s: %#x|%#x [data size: %zu]\n", text,
 250                         msg->primary, msg->extension, msg->data_size);
 251         else
 252                 dev_dbg(dev, "%s: %#x|%#x\n", text, msg->primary, msg->extension);
 253 }
 254 #endif
 255
 256 static void sof_ipc4_dump_payload(struct snd_sof_dev *sdev,
 257                                   void *ipc_data, size_t size)
 258 {
 259         print_hex_dump_debug("Message payload: ", DUMP_PREFIX_OFFSET,
 260                              16, 4, ipc_data, size, false);
 261 }
 262
 263 static int sof_ipc4_get_reply(struct snd_sof_dev *sdev)
 264 {
 265         struct snd_sof_ipc_msg *msg = sdev->msg;
 266         struct sof_ipc4_msg *ipc4_reply;
 267         int ret;
 268
 269         /* get the generic reply */
 270         ipc4_reply = msg->reply_data;
 271
 272         sof_ipc4_log_header(sdev->dev, "ipc tx reply", ipc4_reply, false);
 273
 274         ret = sof_ipc4_check_reply_status(sdev, ipc4_reply->primary);
 275         if (ret)
 276                 return ret;
 277
 278         /* No other information is expected for non large config get replies */
 279         if (!msg->reply_size || !SOF_IPC4_MSG_IS_MODULE_MSG(ipc4_reply->primary) ||
 280             (SOF_IPC4_MSG_TYPE_GET(ipc4_reply->primary) != SOF_IPC4_MOD_LARGE_CONFIG_GET))
 281                 return 0;
 282
 283         /* Read the requested payload */
 284         snd_sof_dsp_mailbox_read(sdev, sdev->dsp_box.offset, ipc4_reply->data_ptr,
 285                                  msg->reply_size);
 286
 287         return 0;
 288 }
 289
 290 /* wait for IPC message reply */
 291 static int ipc4_wait_tx_done(struct snd_sof_ipc *ipc, void *reply_data)
 292 {
 293         struct snd_sof_ipc_msg *msg = &ipc->msg;
 294         struct sof_ipc4_msg *ipc4_msg = msg->msg_data;
 295         struct snd_sof_dev *sdev = ipc->sdev;
 296         int ret;
 297
 298         /* wait for DSP IPC completion */
 299         ret = wait_event_timeout(msg->waitq, msg->ipc_complete,
 300                                  msecs_to_jiffies(sdev->ipc_timeout));
 301         if (ret == 0) {
 302                 dev_err(sdev->dev, "ipc timed out for %#x|%#x\n",
 303                         ipc4_msg->primary, ipc4_msg->extension);
 304                 snd_sof_handle_fw_exception(ipc->sdev, "IPC timeout");
 305                 return -ETIMEDOUT;
 306         }
 307
 308         if (msg->reply_error) {
 309                 dev_err(sdev->dev, "ipc error for msg %#x|%#x\n",
 310                         ipc4_msg->primary, ipc4_msg->extension);
 311                 ret =  msg->reply_error;
 312         } else {
 313                 if (reply_data) {
 314                         struct sof_ipc4_msg *ipc4_reply = msg->reply_data;
 315                         struct sof_ipc4_msg *ipc4_reply_data = reply_data;
 316
 317                         /* Copy the header */
 318                         ipc4_reply_data->header_u64 = ipc4_reply->header_u64;
 319                         if (msg->reply_size && ipc4_reply_data->data_ptr) {
 320                                 /* copy the payload returned from DSP */
 321                                 memcpy(ipc4_reply_data->data_ptr, ipc4_reply->data_ptr,
 322                                        msg->reply_size);
 323                                 ipc4_reply_data->data_size = msg->reply_size;
 324                         }
 325                 }
 326
 327                 ret = 0;
 328                 sof_ipc4_log_header(sdev->dev, "ipc tx done ", ipc4_msg, true);
 329         }
 330
 331         /* re-enable dumps after successful IPC tx */
 332         if (sdev->ipc_dump_printed) {
 333                 sdev->dbg_dump_printed = false;
 334                 sdev->ipc_dump_printed = false;
 335         }
 336
 337         return ret;
 338 }
 339
 340 static int ipc4_tx_msg_unlocked(struct snd_sof_ipc *ipc,
 341                                 void *msg_data, size_t msg_bytes,
 342                                 void *reply_data, size_t reply_bytes)
 343 {
 344         struct sof_ipc4_msg *ipc4_msg = msg_data;
 345         struct snd_sof_dev *sdev = ipc->sdev;
 346         int ret;
 347
 348         if (msg_bytes > ipc->max_payload_size || reply_bytes > ipc->max_payload_size)
 349                 return -EINVAL;
 350
 351         sof_ipc4_log_header(sdev->dev, "ipc tx      ", msg_data, true);
 352
 353         ret = sof_ipc_send_msg(sdev, msg_data, msg_bytes, reply_bytes);
 354         if (ret) {
 355                 dev_err_ratelimited(sdev->dev,
 356                                     "%s: ipc message send for %#x|%#x failed: %d\n",
 357                                     __func__, ipc4_msg->primary, ipc4_msg->extension, ret);
 358                 return ret;
 359         }
 360
 361         /* now wait for completion */
 362         return ipc4_wait_tx_done(ipc, reply_data);
 363 }
 364
 365 static int sof_ipc4_tx_msg(struct snd_sof_dev *sdev, void *msg_data, size_t msg_bytes,
 366                            void *reply_data, size_t reply_bytes, bool no_pm)
 367 {
 368         struct snd_sof_ipc *ipc = sdev->ipc;
 369         int ret;
 370
 371         if (!msg_data)
 372                 return -EINVAL;
 373
 374         if (!no_pm) {
 375                 const struct sof_dsp_power_state target_state = {
 376                         .state = SOF_DSP_PM_D0,
 377                 };
 378
 379                 /* ensure the DSP is in D0i0 before sending a new IPC */
 380                 ret = snd_sof_dsp_set_power_state(sdev, &target_state);
 381                 if (ret < 0)
 382                         return ret;
 383         }
 384
 385         /* Serialise IPC TX */
 386         mutex_lock(&ipc->tx_mutex);
 387
 388         ret = ipc4_tx_msg_unlocked(ipc, msg_data, msg_bytes, reply_data, reply_bytes);
 389
 390         if (sof_debug_check_flag(SOF_DBG_DUMP_IPC_MESSAGE_PAYLOAD)) {
 391                 struct sof_ipc4_msg *msg = NULL;
 392
 393                 /* payload is indicated by non zero msg/reply_bytes */
 394                 if (msg_bytes)
 395                         msg = msg_data;
 396                 else if (reply_bytes)
 397                         msg = reply_data;
 398
 399                 if (msg)
 400                         sof_ipc4_dump_payload(sdev, msg->data_ptr, msg->data_size);
 401         }
 402
 403         mutex_unlock(&ipc->tx_mutex);
 404
 405         return ret;
 406 }
 407
 408 static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data,
 409                                  size_t payload_bytes, bool set)
 410 {
 411         const struct sof_dsp_power_state target_state = {
 412                         .state = SOF_DSP_PM_D0,
 413         };
 414         size_t payload_limit = sdev->ipc->max_payload_size;
 415         struct sof_ipc4_msg *ipc4_msg = data;
 416         struct sof_ipc4_msg tx = {{ 0 }};
 417         struct sof_ipc4_msg rx = {{ 0 }};
 418         size_t remaining = payload_bytes;
 419         size_t offset = 0;
 420         size_t chunk_size;
 421         int ret;
 422
 423         if (!data)
 424                 return -EINVAL;
 425
 426         if ((ipc4_msg->primary & SOF_IPC4_MSG_TARGET_MASK) !=
 427             SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG))
 428                 return -EINVAL;
 429
 430         ipc4_msg->primary &= ~SOF_IPC4_MSG_TYPE_MASK;
 431         tx.primary = ipc4_msg->primary;
 432         tx.extension = ipc4_msg->extension;
 433
 434         if (set)
 435                 tx.primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_SET);
 436         else
 437                 tx.primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_GET);
 438
 439         tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_SIZE_MASK;
 440         tx.extension |= SOF_IPC4_MOD_EXT_MSG_SIZE(payload_bytes);
 441
 442         tx.extension |= SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK(1);
 443
 444         /* ensure the DSP is in D0i0 before sending IPC */
 445         ret = snd_sof_dsp_set_power_state(sdev, &target_state);
 446         if (ret < 0)
 447                 return ret;
 448
 449         /* Serialise IPC TX */
 450         mutex_lock(&sdev->ipc->tx_mutex);
 451
 452         do {
 453                 size_t tx_size, rx_size;
 454
 455                 if (remaining > payload_limit) {
 456                         chunk_size = payload_limit;
 457                 } else {
 458                         chunk_size = remaining;
 459                         if (set)
 460                                 tx.extension |= SOF_IPC4_MOD_EXT_MSG_LAST_BLOCK(1);
 461                 }
 462
 463                 if (offset) {
 464                         tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK_MASK;
 465                         tx.extension &= ~SOF_IPC4_MOD_EXT_MSG_SIZE_MASK;
 466                         tx.extension |= SOF_IPC4_MOD_EXT_MSG_SIZE(offset);
 467                 }
 468
 469                 if (set) {
 470                         tx.data_size = chunk_size;
 471                         tx.data_ptr = ipc4_msg->data_ptr + offset;
 472
 473                         tx_size = chunk_size;
 474                         rx_size = 0;
 475                 } else {
 476                         rx.primary = 0;
 477                         rx.extension = 0;
 478                         rx.data_size = chunk_size;
 479                         rx.data_ptr = ipc4_msg->data_ptr + offset;
 480
 481                         tx_size = 0;
 482                         rx_size = chunk_size;
 483                 }
 484
 485                 /* Send the message for the current chunk */
 486                 ret = ipc4_tx_msg_unlocked(sdev->ipc, &tx, tx_size, &rx, rx_size);
 487                 if (ret < 0) {
 488                         dev_err(sdev->dev,
 489                                 "%s: large config %s failed at offset %zu: %d\n",
 490                                 __func__, set ? "set" : "get", offset, ret);
 491                         goto out;
 492                 }
 493
 494                 if (!set && rx.extension & SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK_MASK) {
 495                         /* Verify the firmware reported total payload size */
 496                         rx_size = rx.extension & SOF_IPC4_MOD_EXT_MSG_SIZE_MASK;
 497
 498                         if (rx_size > payload_bytes) {
 499                                 dev_err(sdev->dev,
 500                                         "%s: Receive buffer (%zu) is too small for %zu\n",
 501                                         __func__, payload_bytes, rx_size);
 502                                 ret = -ENOMEM;
 503                                 goto out;
 504                         }
 505
 506                         if (rx_size < chunk_size) {
 507                                 chunk_size = rx_size;
 508                                 remaining = rx_size;
 509                         } else if (rx_size < payload_bytes) {
 510                                 remaining = rx_size;
 511                         }
 512                 }
 513
 514                 offset += chunk_size;
 515                 remaining -= chunk_size;
 516         } while (remaining);
 517
 518         /* Adjust the received data size if needed */
 519         if (!set && payload_bytes != offset)
 520                 ipc4_msg->data_size = offset;
 521
 522 out:
 523         if (sof_debug_check_flag(SOF_DBG_DUMP_IPC_MESSAGE_PAYLOAD))
 524                 sof_ipc4_dump_payload(sdev, ipc4_msg->data_ptr, ipc4_msg->data_size);
 525
 526         mutex_unlock(&sdev->ipc->tx_mutex);
 527
 528         return ret;
 529 }
 530
 531 static int sof_ipc4_init_msg_memory(struct snd_sof_dev *sdev)
 532 {
 533         struct sof_ipc4_msg *ipc4_msg;
 534         struct snd_sof_ipc_msg *msg = &sdev->ipc->msg;
 535
 536         /* TODO: get max_payload_size from firmware */
 537         sdev->ipc->max_payload_size = SOF_IPC4_MSG_MAX_SIZE;
 538
 539         /* Allocate memory for the ipc4 container and the maximum payload */
 540         msg->reply_data = devm_kzalloc(sdev->dev, sdev->ipc->max_payload_size +
 541                                        sizeof(struct sof_ipc4_msg), GFP_KERNEL);
 542         if (!msg->reply_data)
 543                 return -ENOMEM;
 544
 545         ipc4_msg = msg->reply_data;
 546         ipc4_msg->data_ptr = msg->reply_data + sizeof(struct sof_ipc4_msg);
 547
 548         return 0;
 549 }
 550
 551 size_t sof_ipc4_find_debug_slot_offset_by_type(struct snd_sof_dev *sdev,
 552                                                u32 slot_type)
 553 {
 554         size_t slot_desc_type_offset;
 555         u32 type;
 556         int i;
 557
 558         /* The type is the second u32 in the slot descriptor */
 559         slot_desc_type_offset = sdev->debug_box.offset + sizeof(u32);
 560         for (i = 0; i < SOF_IPC4_MAX_DEBUG_SLOTS; i++) {
 561                 sof_mailbox_read(sdev, slot_desc_type_offset, &type, sizeof(type));
 562
 563                 if (type == slot_type)
 564                         return sdev->debug_box.offset + (i + 1) * SOF_IPC4_DEBUG_SLOT_SIZE;
 565
 566                 slot_desc_type_offset += SOF_IPC4_DEBUG_DESCRIPTOR_SIZE;
 567         }
 568
 569         dev_dbg(sdev->dev, "Slot type %#x is not available in debug window\n", slot_type);
 570         return 0;
 571 }
 572 EXPORT_SYMBOL(sof_ipc4_find_debug_slot_offset_by_type);
 573
 574 static int ipc4_fw_ready(struct snd_sof_dev *sdev, struct sof_ipc4_msg *ipc4_msg)
 575 {
 576         int inbox_offset, inbox_size, outbox_offset, outbox_size;
 577
 578         /* no need to re-check version/ABI for subsequent boots */
 579         if (!sdev->first_boot)
 580                 return 0;
 581
 582         /* Set up the windows for IPC communication */
 583         inbox_offset = snd_sof_dsp_get_mailbox_offset(sdev);
 584         if (inbox_offset < 0) {
 585                 dev_err(sdev->dev, "%s: No mailbox offset\n", __func__);
 586                 return inbox_offset;
 587         }
 588         inbox_size = SOF_IPC4_MSG_MAX_SIZE;
 589         outbox_offset = snd_sof_dsp_get_window_offset(sdev, SOF_IPC4_OUTBOX_WINDOW_IDX);
 590         outbox_size = SOF_IPC4_MSG_MAX_SIZE;
 591
 592         sdev->fw_info_box.offset = snd_sof_dsp_get_window_offset(sdev, SOF_IPC4_INBOX_WINDOW_IDX);
 593         sdev->fw_info_box.size = sizeof(struct sof_ipc4_fw_registers);
 594         sdev->dsp_box.offset = inbox_offset;
 595         sdev->dsp_box.size = inbox_size;
 596         sdev->host_box.offset = outbox_offset;
 597         sdev->host_box.size = outbox_size;
 598
 599         sdev->debug_box.offset = snd_sof_dsp_get_window_offset(sdev,
 600                                                         SOF_IPC4_DEBUG_WINDOW_IDX);
 601
 602         sof_ipc4_create_exception_debugfs_node(sdev);
 603
 604         dev_dbg(sdev->dev, "mailbox upstream 0x%x - size 0x%x\n",
 605                 inbox_offset, inbox_size);
 606         dev_dbg(sdev->dev, "mailbox downstream 0x%x - size 0x%x\n",
 607                 outbox_offset, outbox_size);
 608         dev_dbg(sdev->dev, "debug box 0x%x\n", sdev->debug_box.offset);
 609
 610         return sof_ipc4_init_msg_memory(sdev);
 611 }
 612
 613 static void sof_ipc4_rx_msg(struct snd_sof_dev *sdev)
 614 {
 615         struct sof_ipc4_msg *ipc4_msg = sdev->ipc->msg.rx_data;
 616         size_t data_size = 0;
 617         int err;
 618
 619         if (!ipc4_msg || !SOF_IPC4_MSG_IS_NOTIFICATION(ipc4_msg->primary))
 620                 return;
 621
 622         ipc4_msg->data_ptr = NULL;
 623         ipc4_msg->data_size = 0;
 624
 625         sof_ipc4_log_header(sdev->dev, "ipc rx      ", ipc4_msg, false);
 626
 627         switch (SOF_IPC4_NOTIFICATION_TYPE_GET(ipc4_msg->primary)) {
 628         case SOF_IPC4_NOTIFY_FW_READY:
 629                 /* check for FW boot completion */
 630                 if (sdev->fw_state == SOF_FW_BOOT_IN_PROGRESS) {
 631                         err = ipc4_fw_ready(sdev, ipc4_msg);
 632                         if (err < 0)
 633                                 sof_set_fw_state(sdev, SOF_FW_BOOT_READY_FAILED);
 634                         else
 635                                 sof_set_fw_state(sdev, SOF_FW_BOOT_READY_OK);
 636
 637                         /* wake up firmware loader */
 638                         wake_up(&sdev->boot_wait);
 639                 }
 640
 641                 break;
 642         case SOF_IPC4_NOTIFY_RESOURCE_EVENT:
 643                 data_size = sizeof(struct sof_ipc4_notify_resource_data);
 644                 break;
 645         case SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS:
 646                 sof_ipc4_mtrace_update_pos(sdev, SOF_IPC4_LOG_CORE_GET(ipc4_msg->primary));
 647                 break;
 648         case SOF_IPC4_NOTIFY_EXCEPTION_CAUGHT:
 649                 snd_sof_dsp_panic(sdev, 0, true);
 650                 break;
 651         default:
 652                 dev_dbg(sdev->dev, "Unhandled DSP message: %#x|%#x\n",
 653                         ipc4_msg->primary, ipc4_msg->extension);
 654                 break;
 655         }
 656
 657         if (data_size) {
 658                 ipc4_msg->data_ptr = kmalloc(data_size, GFP_KERNEL);
 659                 if (!ipc4_msg->data_ptr)
 660                         return;
 661
 662                 ipc4_msg->data_size = data_size;
 663                 snd_sof_ipc_msg_data(sdev, NULL, ipc4_msg->data_ptr, ipc4_msg->data_size);
 664         }
 665
 666         sof_ipc4_log_header(sdev->dev, "ipc rx done ", ipc4_msg, true);
 667
 668         if (data_size) {
 669                 if (sof_debug_check_flag(SOF_DBG_DUMP_IPC_MESSAGE_PAYLOAD))
 670                         sof_ipc4_dump_payload(sdev, ipc4_msg->data_ptr,
 671                                               ipc4_msg->data_size);
 672
 673                 kfree(ipc4_msg->data_ptr);
 674                 ipc4_msg->data_ptr = NULL;
 675                 ipc4_msg->data_size = 0;
 676         }
 677 }
 678
 679 static int sof_ipc4_set_core_state(struct snd_sof_dev *sdev, int core_idx, bool on)
 680 {
 681         struct sof_ipc4_dx_state_info dx_state;
 682         struct sof_ipc4_msg msg;
 683
 684         dx_state.core_mask = BIT(core_idx);
 685         if (on)
 686                 dx_state.dx_mask = BIT(core_idx);
 687         else
 688                 dx_state.dx_mask = 0;
 689
 690         msg.primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_SET_DX);
 691         msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
 692         msg.primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
 693         msg.extension = 0;
 694         msg.data_ptr = &dx_state;
 695         msg.data_size = sizeof(dx_state);
 696
 697         return sof_ipc4_tx_msg(sdev, &msg, msg.data_size, NULL, 0, false);
 698 }
 699
 700 /*
 701  * The context save callback is used to send a message to the firmware notifying
 702  * it that the primary core is going to be turned off, which is used as an
 703  * indication to prepare for a full power down, thus preparing for IMR boot
 704  * (when supported)
 705  *
 706  * Note: in IPC4 there is no message used to restore context, thus no context
 707  * restore callback is implemented
 708  */
 709 static int sof_ipc4_ctx_save(struct snd_sof_dev *sdev)
 710 {
 711         return sof_ipc4_set_core_state(sdev, SOF_DSP_PRIMARY_CORE, false);
 712 }
 713
 714 static int sof_ipc4_set_pm_gate(struct snd_sof_dev *sdev, u32 flags)
 715 {
 716         struct sof_ipc4_msg msg = {{0}};
 717
 718         msg.primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_SET_D0IX);
 719         msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
 720         msg.primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
 721         msg.extension = flags;
 722
 723         return sof_ipc4_tx_msg(sdev, &msg, 0, NULL, 0, true);
 724 }
 725
 726 static const struct sof_ipc_pm_ops ipc4_pm_ops = {
 727         .ctx_save = sof_ipc4_ctx_save,
 728         .set_core_state = sof_ipc4_set_core_state,
 729         .set_pm_gate = sof_ipc4_set_pm_gate,
 730 };
 731
 732 static int sof_ipc4_init(struct snd_sof_dev *sdev)
 733 {
 734         struct sof_ipc4_fw_data *ipc4_data = sdev->private;
 735
 736         mutex_init(&ipc4_data->pipeline_state_mutex);
 737
 738         xa_init_flags(&ipc4_data->fw_lib_xa, XA_FLAGS_ALLOC);
 739
 740         return 0;
 741 }
 742
 743 static void sof_ipc4_exit(struct snd_sof_dev *sdev)
 744 {
 745         struct sof_ipc4_fw_data *ipc4_data = sdev->private;
 746         struct sof_ipc4_fw_library *fw_lib;
 747         unsigned long lib_id;
 748
 749         xa_for_each(&ipc4_data->fw_lib_xa, lib_id, fw_lib) {
 750                 /*
 751                  * The basefw (ID == 0) is handled by generic code, it is not
 752                  * loaded by IPC4 code.
 753                  */
 754                 if (lib_id != 0)
 755                         release_firmware(fw_lib->sof_fw.fw);
 756
 757                 fw_lib->sof_fw.fw = NULL;
 758         }
 759
 760         xa_destroy(&ipc4_data->fw_lib_xa);
 761 }
 762
 763 static int sof_ipc4_post_boot(struct snd_sof_dev *sdev)
 764 {
 765         if (sdev->first_boot)
 766                 return sof_ipc4_query_fw_configuration(sdev);
 767
 768         return sof_ipc4_reload_fw_libraries(sdev);
 769 }
 770
 771 const struct sof_ipc_ops ipc4_ops = {
 772         .init = sof_ipc4_init,
 773         .exit = sof_ipc4_exit,
 774         .post_fw_boot = sof_ipc4_post_boot,
 775         .tx_msg = sof_ipc4_tx_msg,
 776         .rx_msg = sof_ipc4_rx_msg,
 777         .set_get_data = sof_ipc4_set_get_data,
 778         .get_reply = sof_ipc4_get_reply,
 779         .pm = &ipc4_pm_ops,
 780         .fw_loader = &ipc4_loader_ops,
 781         .tplg = &ipc4_tplg_ops,
 782         .pcm = &ipc4_pcm_ops,
 783         .fw_tracing = &ipc4_mtrace_ops,
 784 };