1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include "habanalabs.h"
9 #include "../include/common/hl_boot_if.h"
11 #include <linux/firmware.h>
12 #include <linux/genalloc.h>
13 #include <linux/io-64-nonatomic-lo-hi.h>
14 #include <linux/slab.h>
16 #define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */
18 * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
20 * @hdev: pointer to hl_device structure.
21 * @fw_name: the firmware image name
22 * @dst: IO memory mapped address space to copy firmware to
24 * Copy fw code from firmware file to device memory.
26 * Return: 0 on success, non-zero for failure.
28 int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
31 const struct firmware *fw;
36 rc = reject_firmware(&fw, fw_name, hdev->dev);
38 dev_err(hdev->dev, "Firmware file %s is not found!\n", fw_name);
43 if ((fw_size % 4) != 0) {
44 dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
50 dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
52 if (fw_size > FW_FILE_MAX_SIZE) {
54 "FW file size %zu exceeds maximum of %u bytes\n",
55 fw_size, FW_FILE_MAX_SIZE);
60 fw_data = (const u64 *) fw->data;
62 memcpy_toio(dst, fw_data, fw_size);
69 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
71 struct cpucp_packet pkt = {};
73 pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);
75 return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
76 sizeof(pkt), 0, NULL);
79 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
80 u16 len, u32 timeout, long *result)
82 struct cpucp_packet *pkt;
83 dma_addr_t pkt_dma_addr;
87 pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
91 "Failed to allocate DMA memory for packet to CPU\n");
95 memcpy(pkt, msg, len);
97 mutex_lock(&hdev->send_cpu_message_lock);
102 if (hdev->device_cpu_disabled) {
107 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr);
109 dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc);
113 rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,
114 (tmp == CPUCP_PACKET_FENCE_VAL), 1000,
117 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
119 if (rc == -ETIMEDOUT) {
120 dev_err(hdev->dev, "Device CPU packet timeout (0x%x)\n", tmp);
121 hdev->device_cpu_disabled = true;
125 tmp = le32_to_cpu(pkt->ctl);
127 rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;
129 dev_err(hdev->dev, "F/W ERROR %d for CPU packet %d\n",
131 (tmp & CPUCP_PKT_CTL_OPCODE_MASK)
132 >> CPUCP_PKT_CTL_OPCODE_SHIFT);
135 *result = (long) le64_to_cpu(pkt->result);
139 mutex_unlock(&hdev->send_cpu_message_lock);
141 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
146 int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
148 struct cpucp_packet pkt;
152 memset(&pkt, 0, sizeof(pkt));
154 pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
155 CPUCP_PKT_CTL_OPCODE_SHIFT);
156 pkt.value = cpu_to_le64(event_type);
158 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
162 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
167 int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
170 struct cpucp_unmask_irq_arr_packet *pkt;
171 size_t total_pkt_size;
175 total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
178 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
179 total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
181 /* total_pkt_size is casted to u16 later on */
182 if (total_pkt_size > USHRT_MAX) {
183 dev_err(hdev->dev, "too many elements in IRQ array\n");
187 pkt = kzalloc(total_pkt_size, GFP_KERNEL);
191 pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
192 memcpy(&pkt->irqs, irq_arr, irq_arr_size);
194 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
195 CPUCP_PKT_CTL_OPCODE_SHIFT);
197 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
198 total_pkt_size, 0, &result);
201 dev_err(hdev->dev, "failed to unmask IRQ array\n");
208 int hl_fw_test_cpu_queue(struct hl_device *hdev)
210 struct cpucp_packet test_pkt = {};
214 test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
215 CPUCP_PKT_CTL_OPCODE_SHIFT);
216 test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
218 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
219 sizeof(test_pkt), 0, &result);
222 if (result != CPUCP_PACKET_FENCE_VAL)
224 "CPU queue test failed (0x%08lX)\n", result);
226 dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
232 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
233 dma_addr_t *dma_handle)
237 kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
239 *dma_handle = hdev->cpu_accessible_dma_address +
240 (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
242 return (void *) (uintptr_t) kernel_addr;
245 void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
248 gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
252 int hl_fw_send_heartbeat(struct hl_device *hdev)
254 struct cpucp_packet hb_pkt = {};
258 hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
259 CPUCP_PKT_CTL_OPCODE_SHIFT);
260 hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
262 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
263 sizeof(hb_pkt), 0, &result);
265 if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
271 int hl_fw_cpucp_info_get(struct hl_device *hdev)
273 struct asic_fixed_properties *prop = &hdev->asic_prop;
274 struct cpucp_packet pkt = {};
275 void *cpucp_info_cpu_addr;
276 dma_addr_t cpucp_info_dma_addr;
280 cpucp_info_cpu_addr =
281 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
282 sizeof(struct cpucp_info),
283 &cpucp_info_dma_addr);
284 if (!cpucp_info_cpu_addr) {
286 "Failed to allocate DMA memory for CPU-CP info packet\n");
290 memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));
292 pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<
293 CPUCP_PKT_CTL_OPCODE_SHIFT);
294 pkt.addr = cpu_to_le64(cpucp_info_dma_addr);
295 pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));
297 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
298 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
301 "Failed to handle CPU-CP info pkt, error %d\n", rc);
305 memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,
306 sizeof(prop->cpucp_info));
308 rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);
311 "Failed to build hwmon channel info, error %d\n", rc);
317 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
318 sizeof(struct cpucp_info), cpucp_info_cpu_addr);
323 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
325 struct cpucp_packet pkt = {};
326 void *eeprom_info_cpu_addr;
327 dma_addr_t eeprom_info_dma_addr;
331 eeprom_info_cpu_addr =
332 hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
333 max_size, &eeprom_info_dma_addr);
334 if (!eeprom_info_cpu_addr) {
336 "Failed to allocate DMA memory for CPU-CP EEPROM packet\n");
340 memset(eeprom_info_cpu_addr, 0, max_size);
342 pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<
343 CPUCP_PKT_CTL_OPCODE_SHIFT);
344 pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
345 pkt.data_max_size = cpu_to_le32(max_size);
347 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
348 HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);
352 "Failed to handle CPU-CP EEPROM packet, error %d\n",
357 /* result contains the actual size */
358 memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
361 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, max_size,
362 eeprom_info_cpu_addr);
367 int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
368 struct hl_info_pci_counters *counters)
370 struct cpucp_packet pkt = {};
374 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
375 CPUCP_PKT_CTL_OPCODE_SHIFT);
377 /* Fetch PCI rx counter */
378 pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);
379 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
380 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
383 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
386 counters->rx_throughput = result;
388 memset(&pkt, 0, sizeof(pkt));
389 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<
390 CPUCP_PKT_CTL_OPCODE_SHIFT);
392 /* Fetch PCI tx counter */
393 pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);
394 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
395 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
398 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
401 counters->tx_throughput = result;
403 /* Fetch PCI replay counter */
404 memset(&pkt, 0, sizeof(pkt));
405 pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<
406 CPUCP_PKT_CTL_OPCODE_SHIFT);
408 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
409 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
412 "Failed to handle CPU-CP PCI info pkt, error %d\n", rc);
415 counters->replay_cnt = (u32) result;
420 int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)
422 struct cpucp_packet pkt = {};
426 pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<
427 CPUCP_PKT_CTL_OPCODE_SHIFT);
429 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
430 HL_CPUCP_INFO_TIMEOUT_USEC, &result);
433 "Failed to handle CpuCP total energy pkt, error %d\n",
438 *total_energy = result;
443 static void fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg)
447 /* Some of the firmware status codes are deprecated in newer f/w
448 * versions. In those versions, the errors are reported
449 * in different registers. Therefore, we need to check those
450 * registers and print the exact errors. Moreover, there
451 * may be multiple errors, so we need to report on each error
452 * separately. Some of the error codes might indicate a state
453 * that is not an error per-se, but it is an error in production
456 err_val = RREG32(boot_err0_reg);
457 if (!(err_val & CPU_BOOT_ERR0_ENABLED))
460 if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL)
462 "Device boot error - DRAM initialization failed\n");
463 if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED)
464 dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
465 if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL)
467 "Device boot error - Thermal Sensor initialization failed\n");
468 if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED)
470 "Device boot warning - Skipped DRAM initialization\n");
471 if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED)
473 "Device boot error - Skipped waiting for BMC\n");
474 if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY)
476 "Device boot error - Serdes data from BMC not available\n");
477 if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL)
479 "Device boot error - NIC F/W initialization failed\n");
482 static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
484 /* Some of the status codes below are deprecated in newer f/w
485 * versions but we keep them here for backward compatibility
488 case CPU_BOOT_STATUS_NA:
490 "Device boot error - BTL did NOT run\n");
492 case CPU_BOOT_STATUS_IN_WFE:
494 "Device boot error - Stuck inside WFE loop\n");
496 case CPU_BOOT_STATUS_IN_BTL:
498 "Device boot error - Stuck in BTL\n");
500 case CPU_BOOT_STATUS_IN_PREBOOT:
502 "Device boot error - Stuck in Preboot\n");
504 case CPU_BOOT_STATUS_IN_SPL:
506 "Device boot error - Stuck in SPL\n");
508 case CPU_BOOT_STATUS_IN_UBOOT:
510 "Device boot error - Stuck in u-boot\n");
512 case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
514 "Device boot error - DRAM initialization failed\n");
516 case CPU_BOOT_STATUS_UBOOT_NOT_READY:
518 "Device boot error - u-boot stopped by user\n");
520 case CPU_BOOT_STATUS_TS_INIT_FAIL:
522 "Device boot error - Thermal Sensor initialization failed\n");
526 "Device boot error - Invalid status code %d\n",
532 int hl_fw_read_preboot_ver(struct hl_device *hdev, u32 cpu_boot_status_reg,
533 u32 boot_err0_reg, u32 timeout)
538 if (!hdev->cpu_enable)
541 /* Need to check two possible scenarios:
543 * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
544 * the preboot is waiting for the boot fit
546 * All other status values - for older firmwares where the uboot was
547 * loaded from the FLASH
549 rc = hl_poll_timeout(
553 (status == CPU_BOOT_STATUS_IN_UBOOT) ||
554 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
555 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
556 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
557 (status == CPU_BOOT_STATUS_SRAM_AVAIL) ||
558 (status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
563 dev_err(hdev->dev, "Failed to read preboot version\n");
564 detect_cpu_boot_status(hdev, status);
565 fw_read_errors(hdev, boot_err0_reg);
569 hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_PREBOOT);
574 int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
575 u32 msg_to_cpu_reg, u32 cpu_msg_status_reg,
576 u32 boot_err0_reg, bool skip_bmc,
577 u32 cpu_timeout, u32 boot_fit_timeout)
582 dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
583 cpu_timeout / USEC_PER_SEC);
585 /* Wait for boot FIT request */
586 rc = hl_poll_timeout(
590 status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
596 "No boot fit request received, resuming boot\n");
598 rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
602 /* Clear device CPU message status */
603 WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
605 /* Signal device CPU that boot loader is ready */
606 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
608 /* Poll for CPU device ack */
609 rc = hl_poll_timeout(
613 status == CPU_MSG_OK,
619 "Timeout waiting for boot fit load ack\n");
624 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
627 /* Make sure CPU boot-loader is running */
628 rc = hl_poll_timeout(
632 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
633 (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
634 (status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
635 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
639 /* Read U-Boot version now in case we will later fail */
640 hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_UBOOT);
643 detect_cpu_boot_status(hdev, status);
648 if (!hdev->fw_loading) {
649 dev_info(hdev->dev, "Skip loading FW\n");
653 if (status == CPU_BOOT_STATUS_SRAM_AVAIL)
657 "Loading firmware to device, may take some time...\n");
659 rc = hdev->asic_funcs->load_firmware_to_device(hdev);
664 WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
666 rc = hl_poll_timeout(
670 (status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
676 "Failed to get ACK on skipping BMC, %d\n",
678 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
684 WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
686 rc = hl_poll_timeout(
690 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
695 WREG32(msg_to_cpu_reg, KMD_MSG_NA);
698 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
700 "Device reports FIT image is corrupted\n");
703 "Failed to load firmware to device, %d\n",
710 dev_info(hdev->dev, "Successfully loaded firmware to device\n");
713 fw_read_errors(hdev, boot_err0_reg);