2 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
3 * Copyright (C) 2017 Linaro Ltd.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 and
7 * only version 2 as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
16 #include <linux/delay.h>
17 #include <linux/device.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/interrupt.h>
20 #include <linux/iopoll.h>
21 #include <linux/kernel.h>
22 #include <linux/qcom_scm.h>
23 #include <linux/slab.h>
28 #include "hfi_venus.h"
29 #include "hfi_venus_io.h"
31 #define HFI_MASK_QHDR_TX_TYPE 0xff000000
32 #define HFI_MASK_QHDR_RX_TYPE 0x00ff0000
33 #define HFI_MASK_QHDR_PRI_TYPE 0x0000ff00
34 #define HFI_MASK_QHDR_ID_TYPE 0x000000ff
36 #define HFI_HOST_TO_CTRL_CMD_Q 0
37 #define HFI_CTRL_TO_HOST_MSG_Q 1
38 #define HFI_CTRL_TO_HOST_DBG_Q 2
39 #define HFI_MASK_QHDR_STATUS 0x000000ff
42 #define IFACEQ_CMD_IDX 0
43 #define IFACEQ_MSG_IDX 1
44 #define IFACEQ_DBG_IDX 2
45 #define IFACEQ_MAX_BUF_COUNT 50
46 #define IFACEQ_MAX_PARALLEL_CLNTS 16
47 #define IFACEQ_DFLT_QHDR 0x01010000
49 #define POLL_INTERVAL_US 50
51 #define IFACEQ_MAX_PKT_SIZE 1024
52 #define IFACEQ_MED_PKT_SIZE 768
53 #define IFACEQ_MIN_PKT_SIZE 8
54 #define IFACEQ_VAR_SMALL_PKT_SIZE 100
55 #define IFACEQ_VAR_LARGE_PKT_SIZE 512
56 #define IFACEQ_VAR_HUGE_PKT_SIZE (1024 * 12)
58 enum tzbsp_video_state {
59 TZBSP_VIDEO_STATE_SUSPEND = 0,
60 TZBSP_VIDEO_STATE_RESUME
63 struct hfi_queue_table_header {
72 struct hfi_queue_header {
89 #define IFACEQ_TABLE_SIZE \
90 (sizeof(struct hfi_queue_table_header) + \
91 sizeof(struct hfi_queue_header) * IFACEQ_NUM)
93 #define IFACEQ_QUEUE_SIZE (IFACEQ_MAX_PKT_SIZE * \
94 IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)
96 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i) \
97 (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) + \
98 ((i) * sizeof(struct hfi_queue_header)))
100 #define QDSS_SIZE SZ_4K
101 #define SFR_SIZE SZ_4K
103 (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))
105 #define ALIGNED_QDSS_SIZE ALIGN(QDSS_SIZE, SZ_4K)
106 #define ALIGNED_SFR_SIZE ALIGN(SFR_SIZE, SZ_4K)
107 #define ALIGNED_QUEUE_SIZE ALIGN(QUEUE_SIZE, SZ_4K)
108 #define SHARED_QSIZE ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \
109 ALIGNED_QDSS_SIZE, SZ_1M)
112 dma_addr_t da; /* device address */
113 void *kva; /* kernel virtual address */
119 struct hfi_queue_header *qhdr;
120 struct mem_desc qmem;
124 VENUS_STATE_DEINIT = 1,
128 struct venus_hfi_device {
129 struct venus_core *core;
131 u32 last_packet_type;
134 enum venus_state state;
135 /* serialize read / write to the shared memory */
137 struct completion pwr_collapse_prep;
138 struct completion release_resource;
139 struct mem_desc ifaceq_table;
141 struct iface_queue queues[IFACEQ_NUM];
142 u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
143 u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
146 static bool venus_pkt_debug;
147 static int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
148 static bool venus_sys_idle_indicator;
149 static bool venus_fw_low_power_mode = true;
150 static int venus_hw_rsp_timeout = 1000;
151 static bool venus_fw_coverage;
153 static void venus_set_state(struct venus_hfi_device *hdev,
154 enum venus_state state)
156 mutex_lock(&hdev->lock);
158 mutex_unlock(&hdev->lock);
161 static bool venus_is_valid_state(struct venus_hfi_device *hdev)
163 return hdev->state != VENUS_STATE_DEINIT;
166 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)
168 size_t pkt_size = *(u32 *)packet;
170 if (!venus_pkt_debug)
173 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,
177 static int venus_write_queue(struct venus_hfi_device *hdev,
178 struct iface_queue *queue,
179 void *packet, u32 *rx_req)
181 struct hfi_queue_header *qhdr;
182 u32 dwords, new_wr_idx;
183 u32 empty_space, rd_idx, wr_idx, qsize;
186 if (!queue->qmem.kva)
193 venus_dump_packet(hdev, packet);
195 dwords = (*(u32 *)packet) >> 2;
199 rd_idx = qhdr->read_idx;
200 wr_idx = qhdr->write_idx;
201 qsize = qhdr->q_size;
202 /* ensure rd/wr indices's are read from memory */
205 if (wr_idx >= rd_idx)
206 empty_space = qsize - (wr_idx - rd_idx);
208 empty_space = rd_idx - wr_idx;
210 if (empty_space <= dwords) {
212 /* ensure tx_req is updated in memory */
218 /* ensure tx_req is updated in memory */
221 new_wr_idx = wr_idx + dwords;
222 wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));
223 if (new_wr_idx < qsize) {
224 memcpy(wr_ptr, packet, dwords << 2);
229 len = (dwords - new_wr_idx) << 2;
230 memcpy(wr_ptr, packet, len);
231 memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);
234 /* make sure packet is written before updating the write index */
237 qhdr->write_idx = new_wr_idx;
238 *rx_req = qhdr->rx_req ? 1 : 0;
240 /* make sure write index is updated before an interrupt is raised */
246 static int venus_read_queue(struct venus_hfi_device *hdev,
247 struct iface_queue *queue, void *pkt, u32 *tx_req)
249 struct hfi_queue_header *qhdr;
250 u32 dwords, new_rd_idx;
251 u32 rd_idx, wr_idx, type, qsize;
253 u32 recv_request = 0;
256 if (!queue->qmem.kva)
264 rd_idx = qhdr->read_idx;
265 wr_idx = qhdr->write_idx;
266 qsize = qhdr->q_size;
268 /* make sure data is valid before using it */
272 * Do not set receive request for debug queue, if set, Venus generates
273 * interrupt for debug messages even when there is no response message
274 * available. In general debug queue will not become full as it is being
275 * emptied out for every interrupt from Venus. Venus will anyway
276 * generates interrupt if it is full.
278 if (type & HFI_CTRL_TO_HOST_MSG_Q)
281 if (rd_idx == wr_idx) {
282 qhdr->rx_req = recv_request;
284 /* update rx_req field in memory */
289 rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));
290 dwords = *rd_ptr >> 2;
294 new_rd_idx = rd_idx + dwords;
295 if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {
296 if (new_rd_idx < qsize) {
297 memcpy(pkt, rd_ptr, dwords << 2);
302 len = (dwords - new_rd_idx) << 2;
303 memcpy(pkt, rd_ptr, len);
304 memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);
307 /* bad packet received, dropping */
308 new_rd_idx = qhdr->write_idx;
312 /* ensure the packet is read before updating read index */
315 qhdr->read_idx = new_rd_idx;
316 /* ensure updating read index */
319 rd_idx = qhdr->read_idx;
320 wr_idx = qhdr->write_idx;
321 /* ensure rd/wr indices are read from memory */
324 if (rd_idx != wr_idx)
327 qhdr->rx_req = recv_request;
329 *tx_req = qhdr->tx_req ? 1 : 0;
331 /* ensure rx_req is stored to memory and tx_req is loaded from memory */
334 venus_dump_packet(hdev, pkt);
339 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,
342 struct device *dev = hdev->core->dev;
344 desc->attrs = DMA_ATTR_WRITE_COMBINE;
345 desc->size = ALIGN(size, SZ_4K);
347 desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,
355 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)
357 struct device *dev = hdev->core->dev;
359 dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);
362 static void venus_writel(struct venus_hfi_device *hdev, u32 reg, u32 value)
364 writel(value, hdev->core->base + reg);
367 static u32 venus_readl(struct venus_hfi_device *hdev, u32 reg)
369 return readl(hdev->core->base + reg);
372 static void venus_set_registers(struct venus_hfi_device *hdev)
374 const struct venus_resources *res = hdev->core->res;
375 const struct reg_val *tbl = res->reg_tbl;
376 unsigned int count = res->reg_tbl_size;
379 for (i = 0; i < count; i++)
380 venus_writel(hdev, tbl[i].reg, tbl[i].value);
383 static void venus_soft_int(struct venus_hfi_device *hdev)
385 venus_writel(hdev, CPU_IC_SOFTINT, BIT(CPU_IC_SOFTINT_H2A_SHIFT));
388 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,
391 struct device *dev = hdev->core->dev;
392 struct hfi_pkt_hdr *cmd_packet;
393 struct iface_queue *queue;
397 if (!venus_is_valid_state(hdev))
400 cmd_packet = (struct hfi_pkt_hdr *)pkt;
401 hdev->last_packet_type = cmd_packet->pkt_type;
403 queue = &hdev->queues[IFACEQ_CMD_IDX];
405 ret = venus_write_queue(hdev, queue, pkt, &rx_req);
407 dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);
412 venus_soft_int(hdev);
417 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt)
421 mutex_lock(&hdev->lock);
422 ret = venus_iface_cmdq_write_nolock(hdev, pkt);
423 mutex_unlock(&hdev->lock);
428 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,
429 u32 size, u32 addr, void *cookie)
431 struct venus_hfi_device *hdev = to_hfi_priv(core);
432 struct hfi_sys_set_resource_pkt *pkt;
433 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
436 if (id == VIDC_RESOURCE_NONE)
439 pkt = (struct hfi_sys_set_resource_pkt *)packet;
441 ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);
445 ret = venus_iface_cmdq_write(hdev, pkt);
452 static int venus_boot_core(struct venus_hfi_device *hdev)
454 struct device *dev = hdev->core->dev;
455 static const unsigned int max_tries = 100;
457 unsigned int count = 0;
460 venus_writel(hdev, VIDC_CTRL_INIT, BIT(VIDC_CTRL_INIT_CTRL_SHIFT));
461 venus_writel(hdev, WRAPPER_INTR_MASK, WRAPPER_INTR_MASK_A2HVCODEC_MASK);
462 venus_writel(hdev, CPU_CS_SCIACMDARG3, 1);
464 while (!ctrl_status && count < max_tries) {
465 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
466 if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {
467 dev_err(dev, "invalid setting for UC_REGION\n");
472 usleep_range(500, 1000);
476 if (count >= max_tries)
482 static u32 venus_hwversion(struct venus_hfi_device *hdev)
484 struct device *dev = hdev->core->dev;
485 u32 ver = venus_readl(hdev, WRAPPER_HW_VERSION);
486 u32 major, minor, step;
488 major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;
489 major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;
490 minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;
491 minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
492 step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;
494 dev_dbg(dev, "venus hw version %x.%x.%x\n", major, minor, step);
499 static int venus_run(struct venus_hfi_device *hdev)
501 struct device *dev = hdev->core->dev;
505 * Re-program all of the registers that get reset as a result of
506 * regulator_disable() and _enable()
508 venus_set_registers(hdev);
510 venus_writel(hdev, UC_REGION_ADDR, hdev->ifaceq_table.da);
511 venus_writel(hdev, UC_REGION_SIZE, SHARED_QSIZE);
512 venus_writel(hdev, CPU_CS_SCIACMDARG2, hdev->ifaceq_table.da);
513 venus_writel(hdev, CPU_CS_SCIACMDARG1, 0x01);
515 venus_writel(hdev, SFR_ADDR, hdev->sfr.da);
517 ret = venus_boot_core(hdev);
519 dev_err(dev, "failed to reset venus core\n");
523 venus_hwversion(hdev);
528 static int venus_halt_axi(struct venus_hfi_device *hdev)
530 void __iomem *base = hdev->core->base;
531 struct device *dev = hdev->core->dev;
535 if (IS_V4(hdev->core)) {
536 val = venus_readl(hdev, WRAPPER_CPU_AXI_HALT);
537 val |= WRAPPER_CPU_AXI_HALT_HALT;
538 venus_writel(hdev, WRAPPER_CPU_AXI_HALT, val);
540 ret = readl_poll_timeout(base + WRAPPER_CPU_AXI_HALT_STATUS,
542 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,
544 VBIF_AXI_HALT_ACK_TIMEOUT_US);
546 dev_err(dev, "AXI bus port halt timeout\n");
553 /* Halt AXI and AXI IMEM VBIF Access */
554 val = venus_readl(hdev, VBIF_AXI_HALT_CTRL0);
555 val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;
556 venus_writel(hdev, VBIF_AXI_HALT_CTRL0, val);
558 /* Request for AXI bus port halt */
559 ret = readl_poll_timeout(base + VBIF_AXI_HALT_CTRL1, val,
560 val & VBIF_AXI_HALT_CTRL1_HALT_ACK,
562 VBIF_AXI_HALT_ACK_TIMEOUT_US);
564 dev_err(dev, "AXI bus port halt timeout\n");
571 static int venus_power_off(struct venus_hfi_device *hdev)
575 if (!hdev->power_enabled)
578 ret = qcom_scm_set_remote_state(TZBSP_VIDEO_STATE_SUSPEND, 0);
582 ret = venus_halt_axi(hdev);
586 hdev->power_enabled = false;
591 static int venus_power_on(struct venus_hfi_device *hdev)
595 if (hdev->power_enabled)
598 ret = qcom_scm_set_remote_state(TZBSP_VIDEO_STATE_RESUME, 0);
602 ret = venus_run(hdev);
606 hdev->power_enabled = true;
611 qcom_scm_set_remote_state(TZBSP_VIDEO_STATE_SUSPEND, 0);
613 hdev->power_enabled = false;
617 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,
620 struct iface_queue *queue;
624 if (!venus_is_valid_state(hdev))
627 queue = &hdev->queues[IFACEQ_MSG_IDX];
629 ret = venus_read_queue(hdev, queue, pkt, &tx_req);
634 venus_soft_int(hdev);
639 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)
643 mutex_lock(&hdev->lock);
644 ret = venus_iface_msgq_read_nolock(hdev, pkt);
645 mutex_unlock(&hdev->lock);
650 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,
653 struct iface_queue *queue;
657 ret = venus_is_valid_state(hdev);
661 queue = &hdev->queues[IFACEQ_DBG_IDX];
663 ret = venus_read_queue(hdev, queue, pkt, &tx_req);
668 venus_soft_int(hdev);
673 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)
680 mutex_lock(&hdev->lock);
681 ret = venus_iface_dbgq_read_nolock(hdev, pkt);
682 mutex_unlock(&hdev->lock);
687 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)
690 qhdr->type = IFACEQ_DFLT_QHDR;
691 qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;
697 qhdr->rx_irq_status = 0;
698 qhdr->tx_irq_status = 0;
703 static void venus_interface_queues_release(struct venus_hfi_device *hdev)
705 mutex_lock(&hdev->lock);
707 venus_free(hdev, &hdev->ifaceq_table);
708 venus_free(hdev, &hdev->sfr);
710 memset(hdev->queues, 0, sizeof(hdev->queues));
711 memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));
712 memset(&hdev->sfr, 0, sizeof(hdev->sfr));
714 mutex_unlock(&hdev->lock);
717 static int venus_interface_queues_init(struct venus_hfi_device *hdev)
719 struct hfi_queue_table_header *tbl_hdr;
720 struct iface_queue *queue;
722 struct mem_desc desc = {0};
727 ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);
731 hdev->ifaceq_table = desc;
732 offset = IFACEQ_TABLE_SIZE;
734 for (i = 0; i < IFACEQ_NUM; i++) {
735 queue = &hdev->queues[i];
736 queue->qmem.da = desc.da + offset;
737 queue->qmem.kva = desc.kva + offset;
738 queue->qmem.size = IFACEQ_QUEUE_SIZE;
739 offset += queue->qmem.size;
741 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
743 venus_set_qhdr_defaults(queue->qhdr);
745 queue->qhdr->start_addr = queue->qmem.da;
747 if (i == IFACEQ_CMD_IDX)
748 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
749 else if (i == IFACEQ_MSG_IDX)
750 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
751 else if (i == IFACEQ_DBG_IDX)
752 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
755 tbl_hdr = hdev->ifaceq_table.kva;
756 tbl_hdr->version = 0;
757 tbl_hdr->size = IFACEQ_TABLE_SIZE;
758 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
759 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
760 tbl_hdr->num_q = IFACEQ_NUM;
761 tbl_hdr->num_active_q = IFACEQ_NUM;
764 * Set receive request to zero on debug queue as there is no
765 * need of interrupt from video hardware for debug messages
767 queue = &hdev->queues[IFACEQ_DBG_IDX];
768 queue->qhdr->rx_req = 0;
770 ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);
776 sfr->buf_size = ALIGNED_SFR_SIZE;
779 /* ensure table and queue header structs are settled in memory */
785 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)
787 struct hfi_sys_set_property_pkt *pkt;
788 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
791 pkt = (struct hfi_sys_set_property_pkt *)packet;
793 pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);
795 ret = venus_iface_cmdq_write(hdev, pkt);
802 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)
804 struct hfi_sys_set_property_pkt *pkt;
805 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
808 pkt = (struct hfi_sys_set_property_pkt *)packet;
810 pkt_sys_coverage_config(pkt, mode);
812 ret = venus_iface_cmdq_write(hdev, pkt);
819 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,
822 struct hfi_sys_set_property_pkt *pkt;
823 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
829 pkt = (struct hfi_sys_set_property_pkt *)packet;
831 pkt_sys_idle_indicator(pkt, enable);
833 ret = venus_iface_cmdq_write(hdev, pkt);
840 static int venus_sys_set_power_control(struct venus_hfi_device *hdev,
843 struct hfi_sys_set_property_pkt *pkt;
844 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
847 pkt = (struct hfi_sys_set_property_pkt *)packet;
849 pkt_sys_power_control(pkt, enable);
851 ret = venus_iface_cmdq_write(hdev, pkt);
858 static int venus_get_queue_size(struct venus_hfi_device *hdev,
861 struct hfi_queue_header *qhdr;
863 if (index >= IFACEQ_NUM)
866 qhdr = hdev->queues[index].qhdr;
870 return abs(qhdr->read_idx - qhdr->write_idx);
873 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)
875 struct device *dev = hdev->core->dev;
878 ret = venus_sys_set_debug(hdev, venus_fw_debug);
880 dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);
883 * Idle indicator is disabled by default on some 4xx firmware versions,
884 * enable it explicitly in order to make suspend functional by checking
885 * WFI (wait-for-interrupt) bit.
887 if (IS_V4(hdev->core))
888 venus_sys_idle_indicator = true;
890 ret = venus_sys_set_idle_message(hdev, venus_sys_idle_indicator);
892 dev_warn(dev, "setting idle response ON failed (%d)\n", ret);
894 ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);
896 dev_warn(dev, "setting hw power collapse ON failed (%d)\n",
902 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type)
904 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
905 struct hfi_session_pkt pkt;
907 pkt_session_cmd(&pkt, pkt_type, inst);
909 return venus_iface_cmdq_write(hdev, &pkt);
912 static void venus_flush_debug_queue(struct venus_hfi_device *hdev)
914 struct device *dev = hdev->core->dev;
915 void *packet = hdev->dbg_buf;
917 while (!venus_iface_dbgq_read(hdev, packet)) {
918 struct hfi_msg_sys_coverage_pkt *pkt = packet;
920 if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
921 struct hfi_msg_sys_debug_pkt *pkt = packet;
923 dev_dbg(dev, "%s", pkt->msg_data);
928 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,
931 unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);
932 struct hfi_sys_pc_prep_pkt pkt;
935 init_completion(&hdev->pwr_collapse_prep);
937 pkt_sys_pc_prep(&pkt);
939 ret = venus_iface_cmdq_write(hdev, &pkt);
946 ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);
948 venus_flush_debug_queue(hdev);
955 static int venus_are_queues_empty(struct venus_hfi_device *hdev)
959 ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);
963 ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);
973 static void venus_sfr_print(struct venus_hfi_device *hdev)
975 struct device *dev = hdev->core->dev;
976 struct hfi_sfr *sfr = hdev->sfr.kva;
982 p = memchr(sfr->data, '\0', sfr->buf_size);
984 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates
985 * that Venus is in the process of crashing.
988 sfr->data[sfr->buf_size - 1] = '\0';
990 dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);
993 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,
996 struct hfi_msg_event_notify_pkt *event_pkt = packet;
998 if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)
1001 venus_set_state(hdev, VENUS_STATE_DEINIT);
1004 * Once SYS_ERROR received from HW, it is safe to halt the AXI.
1005 * With SYS_ERROR, Venus FW may have crashed and HW might be
1006 * active and causing unnecessary transactions. Hence it is
1007 * safe to stop all AXI transactions from venus subsystem.
1009 venus_halt_axi(hdev);
1010 venus_sfr_print(hdev);
1013 static irqreturn_t venus_isr_thread(struct venus_core *core)
1015 struct venus_hfi_device *hdev = to_hfi_priv(core);
1016 const struct venus_resources *res;
1023 res = hdev->core->res;
1024 pkt = hdev->pkt_buf;
1026 if (hdev->irq_status & WRAPPER_INTR_STATUS_A2HWD_MASK) {
1027 venus_sfr_print(hdev);
1028 hfi_process_watchdog_timeout(core);
1031 while (!venus_iface_msgq_read(hdev, pkt)) {
1032 msg_ret = hfi_process_msg_packet(core, pkt);
1034 case HFI_MSG_EVENT_NOTIFY:
1035 venus_process_msg_sys_error(hdev, pkt);
1037 case HFI_MSG_SYS_INIT:
1038 venus_hfi_core_set_resource(core, res->vmem_id,
1043 case HFI_MSG_SYS_RELEASE_RESOURCE:
1044 complete(&hdev->release_resource);
1046 case HFI_MSG_SYS_PC_PREP:
1047 complete(&hdev->pwr_collapse_prep);
1054 venus_flush_debug_queue(hdev);
1059 static irqreturn_t venus_isr(struct venus_core *core)
1061 struct venus_hfi_device *hdev = to_hfi_priv(core);
1067 status = venus_readl(hdev, WRAPPER_INTR_STATUS);
1069 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1070 status & WRAPPER_INTR_STATUS_A2HWD_MASK ||
1071 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1072 hdev->irq_status = status;
1074 venus_writel(hdev, CPU_CS_A2HSOFTINTCLR, 1);
1075 venus_writel(hdev, WRAPPER_INTR_CLEAR, status);
1077 return IRQ_WAKE_THREAD;
1080 static int venus_core_init(struct venus_core *core)
1082 struct venus_hfi_device *hdev = to_hfi_priv(core);
1083 struct device *dev = core->dev;
1084 struct hfi_sys_get_property_pkt version_pkt;
1085 struct hfi_sys_init_pkt pkt;
1088 pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);
1090 venus_set_state(hdev, VENUS_STATE_INIT);
1092 ret = venus_iface_cmdq_write(hdev, &pkt);
1096 pkt_sys_image_version(&version_pkt);
1098 ret = venus_iface_cmdq_write(hdev, &version_pkt);
1100 dev_warn(dev, "failed to send image version pkt to fw\n");
1102 ret = venus_sys_set_default_properties(hdev);
1109 static int venus_core_deinit(struct venus_core *core)
1111 struct venus_hfi_device *hdev = to_hfi_priv(core);
1113 venus_set_state(hdev, VENUS_STATE_DEINIT);
1114 hdev->suspended = true;
1115 hdev->power_enabled = false;
1120 static int venus_core_ping(struct venus_core *core, u32 cookie)
1122 struct venus_hfi_device *hdev = to_hfi_priv(core);
1123 struct hfi_sys_ping_pkt pkt;
1125 pkt_sys_ping(&pkt, cookie);
1127 return venus_iface_cmdq_write(hdev, &pkt);
1130 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)
1132 struct venus_hfi_device *hdev = to_hfi_priv(core);
1133 struct hfi_sys_test_ssr_pkt pkt;
1136 ret = pkt_sys_ssr_cmd(&pkt, trigger_type);
1140 return venus_iface_cmdq_write(hdev, &pkt);
1143 static int venus_session_init(struct venus_inst *inst, u32 session_type,
1146 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1147 struct hfi_session_init_pkt pkt;
1150 ret = pkt_session_init(&pkt, inst, session_type, codec);
1154 ret = venus_iface_cmdq_write(hdev, &pkt);
1161 venus_flush_debug_queue(hdev);
1165 static int venus_session_end(struct venus_inst *inst)
1167 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1168 struct device *dev = hdev->core->dev;
1170 if (venus_fw_coverage) {
1171 if (venus_sys_set_coverage(hdev, venus_fw_coverage))
1172 dev_warn(dev, "fw coverage msg ON failed\n");
1175 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END);
1178 static int venus_session_abort(struct venus_inst *inst)
1180 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1182 venus_flush_debug_queue(hdev);
1184 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT);
1187 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)
1189 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1190 struct hfi_session_flush_pkt pkt;
1193 ret = pkt_session_flush(&pkt, inst, flush_mode);
1197 return venus_iface_cmdq_write(hdev, &pkt);
1200 static int venus_session_start(struct venus_inst *inst)
1202 return venus_session_cmd(inst, HFI_CMD_SESSION_START);
1205 static int venus_session_stop(struct venus_inst *inst)
1207 return venus_session_cmd(inst, HFI_CMD_SESSION_STOP);
1210 static int venus_session_continue(struct venus_inst *inst)
1212 return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE);
1215 static int venus_session_etb(struct venus_inst *inst,
1216 struct hfi_frame_data *in_frame)
1218 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1219 u32 session_type = inst->session_type;
1222 if (session_type == VIDC_SESSION_TYPE_DEC) {
1223 struct hfi_session_empty_buffer_compressed_pkt pkt;
1225 ret = pkt_session_etb_decoder(&pkt, inst, in_frame);
1229 ret = venus_iface_cmdq_write(hdev, &pkt);
1230 } else if (session_type == VIDC_SESSION_TYPE_ENC) {
1231 struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;
1233 ret = pkt_session_etb_encoder(&pkt, inst, in_frame);
1237 ret = venus_iface_cmdq_write(hdev, &pkt);
1245 static int venus_session_ftb(struct venus_inst *inst,
1246 struct hfi_frame_data *out_frame)
1248 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1249 struct hfi_session_fill_buffer_pkt pkt;
1252 ret = pkt_session_ftb(&pkt, inst, out_frame);
1256 return venus_iface_cmdq_write(hdev, &pkt);
1259 static int venus_session_set_buffers(struct venus_inst *inst,
1260 struct hfi_buffer_desc *bd)
1262 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1263 struct hfi_session_set_buffers_pkt *pkt;
1264 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1267 if (bd->buffer_type == HFI_BUFFER_INPUT)
1270 pkt = (struct hfi_session_set_buffers_pkt *)packet;
1272 ret = pkt_session_set_buffers(pkt, inst, bd);
1276 return venus_iface_cmdq_write(hdev, pkt);
1279 static int venus_session_unset_buffers(struct venus_inst *inst,
1280 struct hfi_buffer_desc *bd)
1282 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1283 struct hfi_session_release_buffer_pkt *pkt;
1284 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1287 if (bd->buffer_type == HFI_BUFFER_INPUT)
1290 pkt = (struct hfi_session_release_buffer_pkt *)packet;
1292 ret = pkt_session_unset_buffers(pkt, inst, bd);
1296 return venus_iface_cmdq_write(hdev, pkt);
1299 static int venus_session_load_res(struct venus_inst *inst)
1301 return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES);
1304 static int venus_session_release_res(struct venus_inst *inst)
1306 return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES);
1309 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1312 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1313 struct hfi_session_parse_sequence_header_pkt *pkt;
1314 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1317 pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;
1319 ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);
1323 ret = venus_iface_cmdq_write(hdev, pkt);
1330 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1333 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1334 struct hfi_session_get_sequence_header_pkt *pkt;
1335 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1338 pkt = (struct hfi_session_get_sequence_header_pkt *)packet;
1340 ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);
1344 return venus_iface_cmdq_write(hdev, pkt);
1347 static int venus_session_set_property(struct venus_inst *inst, u32 ptype,
1350 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1351 struct hfi_session_set_property_pkt *pkt;
1352 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1355 pkt = (struct hfi_session_set_property_pkt *)packet;
1357 ret = pkt_session_set_property(pkt, inst, ptype, pdata);
1361 return venus_iface_cmdq_write(hdev, pkt);
1364 static int venus_session_get_property(struct venus_inst *inst, u32 ptype)
1366 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1367 struct hfi_session_get_property_pkt pkt;
1370 ret = pkt_session_get_property(&pkt, inst, ptype);
1374 return venus_iface_cmdq_write(hdev, &pkt);
1377 static int venus_resume(struct venus_core *core)
1379 struct venus_hfi_device *hdev = to_hfi_priv(core);
1382 mutex_lock(&hdev->lock);
1384 if (!hdev->suspended)
1387 ret = venus_power_on(hdev);
1391 hdev->suspended = false;
1393 mutex_unlock(&hdev->lock);
1398 static int venus_suspend_1xx(struct venus_core *core)
1400 struct venus_hfi_device *hdev = to_hfi_priv(core);
1401 struct device *dev = core->dev;
1405 if (!hdev->power_enabled || hdev->suspended)
1408 mutex_lock(&hdev->lock);
1409 ret = venus_is_valid_state(hdev);
1410 mutex_unlock(&hdev->lock);
1413 dev_err(dev, "bad state, cannot suspend\n");
1417 ret = venus_prepare_power_collapse(hdev, true);
1419 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1423 mutex_lock(&hdev->lock);
1425 if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {
1426 mutex_unlock(&hdev->lock);
1430 ret = venus_are_queues_empty(hdev);
1431 if (ret < 0 || !ret) {
1432 mutex_unlock(&hdev->lock);
1436 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
1437 if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {
1438 mutex_unlock(&hdev->lock);
1442 ret = venus_power_off(hdev);
1444 mutex_unlock(&hdev->lock);
1448 hdev->suspended = true;
1450 mutex_unlock(&hdev->lock);
1455 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)
1457 u32 ctrl_status, cpu_status;
1459 cpu_status = venus_readl(hdev, WRAPPER_CPU_STATUS);
1460 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
1462 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1463 ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1469 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)
1471 u32 ctrl_status, cpu_status;
1473 cpu_status = venus_readl(hdev, WRAPPER_CPU_STATUS);
1474 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
1476 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1477 ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1483 static int venus_suspend_3xx(struct venus_core *core)
1485 struct venus_hfi_device *hdev = to_hfi_priv(core);
1486 struct device *dev = core->dev;
1491 if (!hdev->power_enabled || hdev->suspended)
1494 mutex_lock(&hdev->lock);
1495 ret = venus_is_valid_state(hdev);
1496 mutex_unlock(&hdev->lock);
1499 dev_err(dev, "bad state, cannot suspend\n");
1503 ctrl_status = venus_readl(hdev, CPU_CS_SCIACMDARG0);
1504 if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1508 * Power collapse sequence for Venus 3xx and 4xx versions:
1509 * 1. Check for ARM9 and video core to be idle by checking WFI bit
1510 * (bit 0) in CPU status register and by checking Idle (bit 30) in
1511 * Control status register for video core.
1512 * 2. Send a command to prepare for power collapse.
1513 * 3. Check for WFI and PC_READY bits.
1515 ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,
1520 ret = venus_prepare_power_collapse(hdev, false);
1522 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1526 ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,
1532 mutex_lock(&hdev->lock);
1534 ret = venus_power_off(hdev);
1536 dev_err(dev, "venus_power_off (%d)\n", ret);
1537 mutex_unlock(&hdev->lock);
1541 hdev->suspended = true;
1543 mutex_unlock(&hdev->lock);
1548 static int venus_suspend(struct venus_core *core)
1550 if (IS_V3(core) || IS_V4(core))
1551 return venus_suspend_3xx(core);
1553 return venus_suspend_1xx(core);
1556 static const struct hfi_ops venus_hfi_ops = {
1557 .core_init = venus_core_init,
1558 .core_deinit = venus_core_deinit,
1559 .core_ping = venus_core_ping,
1560 .core_trigger_ssr = venus_core_trigger_ssr,
1562 .session_init = venus_session_init,
1563 .session_end = venus_session_end,
1564 .session_abort = venus_session_abort,
1565 .session_flush = venus_session_flush,
1566 .session_start = venus_session_start,
1567 .session_stop = venus_session_stop,
1568 .session_continue = venus_session_continue,
1569 .session_etb = venus_session_etb,
1570 .session_ftb = venus_session_ftb,
1571 .session_set_buffers = venus_session_set_buffers,
1572 .session_unset_buffers = venus_session_unset_buffers,
1573 .session_load_res = venus_session_load_res,
1574 .session_release_res = venus_session_release_res,
1575 .session_parse_seq_hdr = venus_session_parse_seq_hdr,
1576 .session_get_seq_hdr = venus_session_get_seq_hdr,
1577 .session_set_property = venus_session_set_property,
1578 .session_get_property = venus_session_get_property,
1580 .resume = venus_resume,
1581 .suspend = venus_suspend,
1584 .isr_thread = venus_isr_thread,
1587 void venus_hfi_destroy(struct venus_core *core)
1589 struct venus_hfi_device *hdev = to_hfi_priv(core);
1591 venus_interface_queues_release(hdev);
1592 mutex_destroy(&hdev->lock);
1598 int venus_hfi_create(struct venus_core *core)
1600 struct venus_hfi_device *hdev;
1603 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
1607 mutex_init(&hdev->lock);
1610 hdev->suspended = true;
1612 core->ops = &venus_hfi_ops;
1613 core->core_caps = ENC_ROTATION_CAPABILITY | ENC_SCALING_CAPABILITY |
1614 ENC_DEINTERLACE_CAPABILITY |
1615 DEC_MULTI_STREAM_CAPABILITY;
1617 ret = venus_interface_queues_init(hdev);