drivers/media/platform/qcom/venus/hfi_venus.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
   4  * Copyright (C) 2017 Linaro Ltd.
   5  */
   6
   7 #include <linux/delay.h>
   8 #include <linux/device.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/iopoll.h>
  12 #include <linux/kernel.h>
  13 #include <linux/slab.h>
  14
  15 #include "core.h"
  16 #include "hfi_cmds.h"
  17 #include "hfi_msgs.h"
  18 #include "hfi_venus.h"
  19 #include "hfi_venus_io.h"
  20 #include "firmware.h"
  21
  22 #define HFI_MASK_QHDR_TX_TYPE           0xff000000
  23 #define HFI_MASK_QHDR_RX_TYPE           0x00ff0000
  24 #define HFI_MASK_QHDR_PRI_TYPE          0x0000ff00
  25 #define HFI_MASK_QHDR_ID_TYPE           0x000000ff
  26
  27 #define HFI_HOST_TO_CTRL_CMD_Q          0
  28 #define HFI_CTRL_TO_HOST_MSG_Q          1
  29 #define HFI_CTRL_TO_HOST_DBG_Q          2
  30 #define HFI_MASK_QHDR_STATUS            0x000000ff
  31
  32 #define IFACEQ_NUM                      3
  33 #define IFACEQ_CMD_IDX                  0
  34 #define IFACEQ_MSG_IDX                  1
  35 #define IFACEQ_DBG_IDX                  2
  36 #define IFACEQ_MAX_BUF_COUNT            50
  37 #define IFACEQ_MAX_PARALLEL_CLNTS       16
  38 #define IFACEQ_DFLT_QHDR                0x01010000
  39
  40 #define POLL_INTERVAL_US                50
  41
  42 #define IFACEQ_MAX_PKT_SIZE             1024
  43 #define IFACEQ_MED_PKT_SIZE             768
  44 #define IFACEQ_MIN_PKT_SIZE             8
  45 #define IFACEQ_VAR_SMALL_PKT_SIZE       100
  46 #define IFACEQ_VAR_LARGE_PKT_SIZE       512
  47 #define IFACEQ_VAR_HUGE_PKT_SIZE        (1024 * 12)
  48
  49 struct hfi_queue_table_header {
  50         u32 version;
  51         u32 size;
  52         u32 qhdr0_offset;
  53         u32 qhdr_size;
  54         u32 num_q;
  55         u32 num_active_q;
  56 };
  57
  58 struct hfi_queue_header {
  59         u32 status;
  60         u32 start_addr;
  61         u32 type;
  62         u32 q_size;
  63         u32 pkt_size;
  64         u32 pkt_drop_cnt;
  65         u32 rx_wm;
  66         u32 tx_wm;
  67         u32 rx_req;
  68         u32 tx_req;
  69         u32 rx_irq_status;
  70         u32 tx_irq_status;
  71         u32 read_idx;
  72         u32 write_idx;
  73 };
  74
  75 #define IFACEQ_TABLE_SIZE       \
  76         (sizeof(struct hfi_queue_table_header) +        \
  77          sizeof(struct hfi_queue_header) * IFACEQ_NUM)
  78
  79 #define IFACEQ_QUEUE_SIZE       (IFACEQ_MAX_PKT_SIZE *  \
  80         IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)
  81
  82 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i)      \
  83         (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) +      \
  84                 ((i) * sizeof(struct hfi_queue_header)))
  85
  86 #define QDSS_SIZE               SZ_4K
  87 #define SFR_SIZE                SZ_4K
  88 #define QUEUE_SIZE              \
  89         (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))
  90
  91 #define ALIGNED_QDSS_SIZE       ALIGN(QDSS_SIZE, SZ_4K)
  92 #define ALIGNED_SFR_SIZE        ALIGN(SFR_SIZE, SZ_4K)
  93 #define ALIGNED_QUEUE_SIZE      ALIGN(QUEUE_SIZE, SZ_4K)
  94 #define SHARED_QSIZE            ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \
  95                                       ALIGNED_QDSS_SIZE, SZ_1M)
  96
  97 struct mem_desc {
  98         dma_addr_t da;  /* device address */
  99         void *kva;      /* kernel virtual address */
 100         u32 size;
 101         unsigned long attrs;
 102 };
 103
 104 struct iface_queue {
 105         struct hfi_queue_header *qhdr;
 106         struct mem_desc qmem;
 107 };
 108
 109 enum venus_state {
 110         VENUS_STATE_DEINIT = 1,
 111         VENUS_STATE_INIT,
 112 };
 113
 114 struct venus_hfi_device {
 115         struct venus_core *core;
 116         u32 irq_status;
 117         u32 last_packet_type;
 118         bool power_enabled;
 119         bool suspended;
 120         enum venus_state state;
 121         /* serialize read / write to the shared memory */
 122         struct mutex lock;
 123         struct completion pwr_collapse_prep;
 124         struct completion release_resource;
 125         struct mem_desc ifaceq_table;
 126         struct mem_desc sfr;
 127         struct iface_queue queues[IFACEQ_NUM];
 128         u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
 129         u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
 130 };
 131
 132 static bool venus_pkt_debug;
 133 int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
 134 static bool venus_fw_low_power_mode = true;
 135 static int venus_hw_rsp_timeout = 1000;
 136 static bool venus_fw_coverage;
 137
 138 static void venus_set_state(struct venus_hfi_device *hdev,
 139                             enum venus_state state)
 140 {
 141         mutex_lock(&hdev->lock);
 142         hdev->state = state;
 143         mutex_unlock(&hdev->lock);
 144 }
 145
 146 static bool venus_is_valid_state(struct venus_hfi_device *hdev)
 147 {
 148         return hdev->state != VENUS_STATE_DEINIT;
 149 }
 150
 151 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)
 152 {
 153         size_t pkt_size = *(u32 *)packet;
 154
 155         if (!venus_pkt_debug)
 156                 return;
 157
 158         print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,
 159                        pkt_size, true);
 160 }
 161
 162 static int venus_write_queue(struct venus_hfi_device *hdev,
 163                              struct iface_queue *queue,
 164                              void *packet, u32 *rx_req)
 165 {
 166         struct hfi_queue_header *qhdr;
 167         u32 dwords, new_wr_idx;
 168         u32 empty_space, rd_idx, wr_idx, qsize;
 169         u32 *wr_ptr;
 170
 171         if (!queue->qmem.kva)
 172                 return -EINVAL;
 173
 174         qhdr = queue->qhdr;
 175         if (!qhdr)
 176                 return -EINVAL;
 177
 178         venus_dump_packet(hdev, packet);
 179
 180         dwords = (*(u32 *)packet) >> 2;
 181         if (!dwords)
 182                 return -EINVAL;
 183
 184         rd_idx = qhdr->read_idx;
 185         wr_idx = qhdr->write_idx;
 186         qsize = qhdr->q_size;
 187         /* ensure rd/wr indices's are read from memory */
 188         rmb();
 189
 190         if (wr_idx >= rd_idx)
 191                 empty_space = qsize - (wr_idx - rd_idx);
 192         else
 193                 empty_space = rd_idx - wr_idx;
 194
 195         if (empty_space <= dwords) {
 196                 qhdr->tx_req = 1;
 197                 /* ensure tx_req is updated in memory */
 198                 wmb();
 199                 return -ENOSPC;
 200         }
 201
 202         qhdr->tx_req = 0;
 203         /* ensure tx_req is updated in memory */
 204         wmb();
 205
 206         new_wr_idx = wr_idx + dwords;
 207         wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));
 208
 209         if (wr_ptr < (u32 *)queue->qmem.kva ||
 210             wr_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*wr_ptr)))
 211                 return -EINVAL;
 212
 213         if (new_wr_idx < qsize) {
 214                 memcpy(wr_ptr, packet, dwords << 2);
 215         } else {
 216                 size_t len;
 217
 218                 new_wr_idx -= qsize;
 219                 len = (dwords - new_wr_idx) << 2;
 220                 memcpy(wr_ptr, packet, len);
 221                 memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);
 222         }
 223
 224         /* make sure packet is written before updating the write index */
 225         wmb();
 226
 227         qhdr->write_idx = new_wr_idx;
 228         *rx_req = qhdr->rx_req ? 1 : 0;
 229
 230         /* make sure write index is updated before an interrupt is raised */
 231         mb();
 232
 233         return 0;
 234 }
 235
 236 static int venus_read_queue(struct venus_hfi_device *hdev,
 237                             struct iface_queue *queue, void *pkt, u32 *tx_req)
 238 {
 239         struct hfi_queue_header *qhdr;
 240         u32 dwords, new_rd_idx;
 241         u32 rd_idx, wr_idx, type, qsize;
 242         u32 *rd_ptr;
 243         u32 recv_request = 0;
 244         int ret = 0;
 245
 246         if (!queue->qmem.kva)
 247                 return -EINVAL;
 248
 249         qhdr = queue->qhdr;
 250         if (!qhdr)
 251                 return -EINVAL;
 252
 253         type = qhdr->type;
 254         rd_idx = qhdr->read_idx;
 255         wr_idx = qhdr->write_idx;
 256         qsize = qhdr->q_size;
 257
 258         /* make sure data is valid before using it */
 259         rmb();
 260
 261         /*
 262          * Do not set receive request for debug queue, if set, Venus generates
 263          * interrupt for debug messages even when there is no response message
 264          * available. In general debug queue will not become full as it is being
 265          * emptied out for every interrupt from Venus. Venus will anyway
 266          * generates interrupt if it is full.
 267          */
 268         if (type & HFI_CTRL_TO_HOST_MSG_Q)
 269                 recv_request = 1;
 270
 271         if (rd_idx == wr_idx) {
 272                 qhdr->rx_req = recv_request;
 273                 *tx_req = 0;
 274                 /* update rx_req field in memory */
 275                 wmb();
 276                 return -ENODATA;
 277         }
 278
 279         rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));
 280
 281         if (rd_ptr < (u32 *)queue->qmem.kva ||
 282             rd_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*rd_ptr)))
 283                 return -EINVAL;
 284
 285         dwords = *rd_ptr >> 2;
 286         if (!dwords)
 287                 return -EINVAL;
 288
 289         new_rd_idx = rd_idx + dwords;
 290         if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {
 291                 if (new_rd_idx < qsize) {
 292                         memcpy(pkt, rd_ptr, dwords << 2);
 293                 } else {
 294                         size_t len;
 295
 296                         new_rd_idx -= qsize;
 297                         len = (dwords - new_rd_idx) << 2;
 298                         memcpy(pkt, rd_ptr, len);
 299                         memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);
 300                 }
 301         } else {
 302                 /* bad packet received, dropping */
 303                 new_rd_idx = qhdr->write_idx;
 304                 ret = -EBADMSG;
 305         }
 306
 307         /* ensure the packet is read before updating read index */
 308         rmb();
 309
 310         qhdr->read_idx = new_rd_idx;
 311         /* ensure updating read index */
 312         wmb();
 313
 314         rd_idx = qhdr->read_idx;
 315         wr_idx = qhdr->write_idx;
 316         /* ensure rd/wr indices are read from memory */
 317         rmb();
 318
 319         if (rd_idx != wr_idx)
 320                 qhdr->rx_req = 0;
 321         else
 322                 qhdr->rx_req = recv_request;
 323
 324         *tx_req = qhdr->tx_req ? 1 : 0;
 325
 326         /* ensure rx_req is stored to memory and tx_req is loaded from memory */
 327         mb();
 328
 329         venus_dump_packet(hdev, pkt);
 330
 331         return ret;
 332 }
 333
 334 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,
 335                        u32 size)
 336 {
 337         struct device *dev = hdev->core->dev;
 338
 339         desc->attrs = DMA_ATTR_WRITE_COMBINE;
 340         desc->size = ALIGN(size, SZ_4K);
 341
 342         desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,
 343                                     desc->attrs);
 344         if (!desc->kva)
 345                 return -ENOMEM;
 346
 347         return 0;
 348 }
 349
 350 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)
 351 {
 352         struct device *dev = hdev->core->dev;
 353
 354         dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);
 355 }
 356
 357 static void venus_set_registers(struct venus_hfi_device *hdev)
 358 {
 359         const struct venus_resources *res = hdev->core->res;
 360         const struct reg_val *tbl = res->reg_tbl;
 361         unsigned int count = res->reg_tbl_size;
 362         unsigned int i;
 363
 364         for (i = 0; i < count; i++)
 365                 writel(tbl[i].value, hdev->core->base + tbl[i].reg);
 366 }
 367
 368 static void venus_soft_int(struct venus_hfi_device *hdev)
 369 {
 370         void __iomem *cpu_ic_base = hdev->core->cpu_ic_base;
 371         u32 clear_bit;
 372
 373         if (IS_V6(hdev->core))
 374                 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6);
 375         else
 376                 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT);
 377
 378         writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT);
 379 }
 380
 381 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,
 382                                          void *pkt, bool sync)
 383 {
 384         struct device *dev = hdev->core->dev;
 385         struct hfi_pkt_hdr *cmd_packet;
 386         struct iface_queue *queue;
 387         u32 rx_req;
 388         int ret;
 389
 390         if (!venus_is_valid_state(hdev))
 391                 return -EINVAL;
 392
 393         cmd_packet = (struct hfi_pkt_hdr *)pkt;
 394         hdev->last_packet_type = cmd_packet->pkt_type;
 395
 396         queue = &hdev->queues[IFACEQ_CMD_IDX];
 397
 398         ret = venus_write_queue(hdev, queue, pkt, &rx_req);
 399         if (ret) {
 400                 dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);
 401                 return ret;
 402         }
 403
 404         if (sync) {
 405                 /*
 406                  * Inform video hardware to raise interrupt for synchronous
 407                  * commands
 408                  */
 409                 queue = &hdev->queues[IFACEQ_MSG_IDX];
 410                 queue->qhdr->rx_req = 1;
 411                 /* ensure rx_req is updated in memory */
 412                 wmb();
 413         }
 414
 415         if (rx_req)
 416                 venus_soft_int(hdev);
 417
 418         return 0;
 419 }
 420
 421 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync)
 422 {
 423         int ret;
 424
 425         mutex_lock(&hdev->lock);
 426         ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync);
 427         mutex_unlock(&hdev->lock);
 428
 429         return ret;
 430 }
 431
 432 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,
 433                                        u32 size, u32 addr, void *cookie)
 434 {
 435         struct venus_hfi_device *hdev = to_hfi_priv(core);
 436         struct hfi_sys_set_resource_pkt *pkt;
 437         u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
 438         int ret;
 439
 440         if (id == VIDC_RESOURCE_NONE)
 441                 return 0;
 442
 443         pkt = (struct hfi_sys_set_resource_pkt *)packet;
 444
 445         ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);
 446         if (ret)
 447                 return ret;
 448
 449         ret = venus_iface_cmdq_write(hdev, pkt, false);
 450         if (ret)
 451                 return ret;
 452
 453         return 0;
 454 }
 455
 456 static int venus_boot_core(struct venus_hfi_device *hdev)
 457 {
 458         struct device *dev = hdev->core->dev;
 459         static const unsigned int max_tries = 100;
 460         u32 ctrl_status = 0, mask_val = 0;
 461         unsigned int count = 0;
 462         void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
 463         void __iomem *wrapper_base = hdev->core->wrapper_base;
 464         int ret = 0;
 465
 466         if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
 467                 mask_val = readl(wrapper_base + WRAPPER_INTR_MASK);
 468                 mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 |
 469                               WRAPPER_INTR_MASK_A2HCPU_MASK);
 470         } else {
 471                 mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK;
 472         }
 473
 474         writel(mask_val, wrapper_base + WRAPPER_INTR_MASK);
 475         if (IS_V1(hdev->core))
 476                 writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3);
 477
 478         writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT);
 479         while (!ctrl_status && count < max_tries) {
 480                 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
 481                 if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {
 482                         dev_err(dev, "invalid setting for UC_REGION\n");
 483                         ret = -EINVAL;
 484                         break;
 485                 }
 486
 487                 usleep_range(500, 1000);
 488                 count++;
 489         }
 490
 491         if (count >= max_tries)
 492                 ret = -ETIMEDOUT;
 493
 494         if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
 495                 writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6);
 496                 writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6);
 497         }
 498
 499         return ret;
 500 }
 501
 502 static u32 venus_hwversion(struct venus_hfi_device *hdev)
 503 {
 504         struct device *dev = hdev->core->dev;
 505         void __iomem *wrapper_base = hdev->core->wrapper_base;
 506         u32 ver;
 507         u32 major, minor, step;
 508
 509         ver = readl(wrapper_base + WRAPPER_HW_VERSION);
 510         major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;
 511         major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;
 512         minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;
 513         minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
 514         step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;
 515
 516         dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step);
 517
 518         return major;
 519 }
 520
 521 static int venus_run(struct venus_hfi_device *hdev)
 522 {
 523         struct device *dev = hdev->core->dev;
 524         void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
 525         int ret;
 526
 527         /*
 528          * Re-program all of the registers that get reset as a result of
 529          * regulator_disable() and _enable()
 530          */
 531         venus_set_registers(hdev);
 532
 533         writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR);
 534         writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE);
 535         writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2);
 536         writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1);
 537         if (hdev->sfr.da)
 538                 writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR);
 539
 540         ret = venus_boot_core(hdev);
 541         if (ret) {
 542                 dev_err(dev, "failed to reset venus core\n");
 543                 return ret;
 544         }
 545
 546         venus_hwversion(hdev);
 547
 548         return 0;
 549 }
 550
 551 static int venus_halt_axi(struct venus_hfi_device *hdev)
 552 {
 553         void __iomem *wrapper_base = hdev->core->wrapper_base;
 554         void __iomem *vbif_base = hdev->core->vbif_base;
 555         void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
 556         void __iomem *aon_base = hdev->core->aon_base;
 557         struct device *dev = hdev->core->dev;
 558         u32 val;
 559         u32 mask_val;
 560         int ret;
 561
 562         if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
 563                 writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6);
 564
 565                 if (IS_IRIS2_1(hdev->core))
 566                         goto skip_aon_mvp_noc;
 567
 568                 writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL);
 569                 ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS,
 570                                          val,
 571                                          val & BIT(0),
 572                                          POLL_INTERVAL_US,
 573                                          VBIF_AXI_HALT_ACK_TIMEOUT_US);
 574                 if (ret)
 575                         return -ETIMEDOUT;
 576
 577 skip_aon_mvp_noc:
 578                 mask_val = (BIT(2) | BIT(1) | BIT(0));
 579                 writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
 580
 581                 writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
 582                 ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6,
 583                                          val,
 584                                          val == 0,
 585                                          POLL_INTERVAL_US,
 586                                          VBIF_AXI_HALT_ACK_TIMEOUT_US);
 587
 588                 if (ret) {
 589                         dev_err(dev, "DBLP Release: lpi_status %x\n", val);
 590                         return -ETIMEDOUT;
 591                 }
 592                 return 0;
 593         }
 594
 595         if (IS_V4(hdev->core)) {
 596                 val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT);
 597                 val |= WRAPPER_CPU_AXI_HALT_HALT;
 598                 writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT);
 599
 600                 ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS,
 601                                          val,
 602                                          val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,
 603                                          POLL_INTERVAL_US,
 604                                          VBIF_AXI_HALT_ACK_TIMEOUT_US);
 605                 if (ret) {
 606                         dev_err(dev, "AXI bus port halt timeout\n");
 607                         return ret;
 608                 }
 609
 610                 return 0;
 611         }
 612
 613         /* Halt AXI and AXI IMEM VBIF Access */
 614         val = readl(vbif_base + VBIF_AXI_HALT_CTRL0);
 615         val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;
 616         writel(val, vbif_base + VBIF_AXI_HALT_CTRL0);
 617
 618         /* Request for AXI bus port halt */
 619         ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val,
 620                                  val & VBIF_AXI_HALT_CTRL1_HALT_ACK,
 621                                  POLL_INTERVAL_US,
 622                                  VBIF_AXI_HALT_ACK_TIMEOUT_US);
 623         if (ret) {
 624                 dev_err(dev, "AXI bus port halt timeout\n");
 625                 return ret;
 626         }
 627
 628         return 0;
 629 }
 630
 631 static int venus_power_off(struct venus_hfi_device *hdev)
 632 {
 633         int ret;
 634
 635         if (!hdev->power_enabled)
 636                 return 0;
 637
 638         ret = venus_set_hw_state_suspend(hdev->core);
 639         if (ret)
 640                 return ret;
 641
 642         ret = venus_halt_axi(hdev);
 643         if (ret)
 644                 return ret;
 645
 646         hdev->power_enabled = false;
 647
 648         return 0;
 649 }
 650
 651 static int venus_power_on(struct venus_hfi_device *hdev)
 652 {
 653         int ret;
 654
 655         if (hdev->power_enabled)
 656                 return 0;
 657
 658         ret = venus_set_hw_state_resume(hdev->core);
 659         if (ret)
 660                 goto err;
 661
 662         ret = venus_run(hdev);
 663         if (ret)
 664                 goto err_suspend;
 665
 666         hdev->power_enabled = true;
 667
 668         return 0;
 669
 670 err_suspend:
 671         venus_set_hw_state_suspend(hdev->core);
 672 err:
 673         hdev->power_enabled = false;
 674         return ret;
 675 }
 676
 677 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,
 678                                         void *pkt)
 679 {
 680         struct iface_queue *queue;
 681         u32 tx_req;
 682         int ret;
 683
 684         if (!venus_is_valid_state(hdev))
 685                 return -EINVAL;
 686
 687         queue = &hdev->queues[IFACEQ_MSG_IDX];
 688
 689         ret = venus_read_queue(hdev, queue, pkt, &tx_req);
 690         if (ret)
 691                 return ret;
 692
 693         if (tx_req)
 694                 venus_soft_int(hdev);
 695
 696         return 0;
 697 }
 698
 699 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)
 700 {
 701         int ret;
 702
 703         mutex_lock(&hdev->lock);
 704         ret = venus_iface_msgq_read_nolock(hdev, pkt);
 705         mutex_unlock(&hdev->lock);
 706
 707         return ret;
 708 }
 709
 710 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,
 711                                         void *pkt)
 712 {
 713         struct iface_queue *queue;
 714         u32 tx_req;
 715         int ret;
 716
 717         ret = venus_is_valid_state(hdev);
 718         if (!ret)
 719                 return -EINVAL;
 720
 721         queue = &hdev->queues[IFACEQ_DBG_IDX];
 722
 723         ret = venus_read_queue(hdev, queue, pkt, &tx_req);
 724         if (ret)
 725                 return ret;
 726
 727         if (tx_req)
 728                 venus_soft_int(hdev);
 729
 730         return 0;
 731 }
 732
 733 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)
 734 {
 735         int ret;
 736
 737         if (!pkt)
 738                 return -EINVAL;
 739
 740         mutex_lock(&hdev->lock);
 741         ret = venus_iface_dbgq_read_nolock(hdev, pkt);
 742         mutex_unlock(&hdev->lock);
 743
 744         return ret;
 745 }
 746
 747 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)
 748 {
 749         qhdr->status = 1;
 750         qhdr->type = IFACEQ_DFLT_QHDR;
 751         qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;
 752         qhdr->pkt_size = 0;
 753         qhdr->rx_wm = 1;
 754         qhdr->tx_wm = 1;
 755         qhdr->rx_req = 1;
 756         qhdr->tx_req = 0;
 757         qhdr->rx_irq_status = 0;
 758         qhdr->tx_irq_status = 0;
 759         qhdr->read_idx = 0;
 760         qhdr->write_idx = 0;
 761 }
 762
 763 static void venus_interface_queues_release(struct venus_hfi_device *hdev)
 764 {
 765         mutex_lock(&hdev->lock);
 766
 767         venus_free(hdev, &hdev->ifaceq_table);
 768         venus_free(hdev, &hdev->sfr);
 769
 770         memset(hdev->queues, 0, sizeof(hdev->queues));
 771         memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));
 772         memset(&hdev->sfr, 0, sizeof(hdev->sfr));
 773
 774         mutex_unlock(&hdev->lock);
 775 }
 776
 777 static int venus_interface_queues_init(struct venus_hfi_device *hdev)
 778 {
 779         struct hfi_queue_table_header *tbl_hdr;
 780         struct iface_queue *queue;
 781         struct hfi_sfr *sfr;
 782         struct mem_desc desc = {0};
 783         unsigned int offset;
 784         unsigned int i;
 785         int ret;
 786
 787         ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);
 788         if (ret)
 789                 return ret;
 790
 791         hdev->ifaceq_table = desc;
 792         offset = IFACEQ_TABLE_SIZE;
 793
 794         for (i = 0; i < IFACEQ_NUM; i++) {
 795                 queue = &hdev->queues[i];
 796                 queue->qmem.da = desc.da + offset;
 797                 queue->qmem.kva = desc.kva + offset;
 798                 queue->qmem.size = IFACEQ_QUEUE_SIZE;
 799                 offset += queue->qmem.size;
 800                 queue->qhdr =
 801                         IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
 802
 803                 venus_set_qhdr_defaults(queue->qhdr);
 804
 805                 queue->qhdr->start_addr = queue->qmem.da;
 806
 807                 if (i == IFACEQ_CMD_IDX)
 808                         queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
 809                 else if (i == IFACEQ_MSG_IDX)
 810                         queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
 811                 else if (i == IFACEQ_DBG_IDX)
 812                         queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
 813         }
 814
 815         tbl_hdr = hdev->ifaceq_table.kva;
 816         tbl_hdr->version = 0;
 817         tbl_hdr->size = IFACEQ_TABLE_SIZE;
 818         tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
 819         tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
 820         tbl_hdr->num_q = IFACEQ_NUM;
 821         tbl_hdr->num_active_q = IFACEQ_NUM;
 822
 823         /*
 824          * Set receive request to zero on debug queue as there is no
 825          * need of interrupt from video hardware for debug messages
 826          */
 827         queue = &hdev->queues[IFACEQ_DBG_IDX];
 828         queue->qhdr->rx_req = 0;
 829
 830         ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);
 831         if (ret) {
 832                 hdev->sfr.da = 0;
 833         } else {
 834                 hdev->sfr = desc;
 835                 sfr = hdev->sfr.kva;
 836                 sfr->buf_size = ALIGNED_SFR_SIZE;
 837         }
 838
 839         /* ensure table and queue header structs are settled in memory */
 840         wmb();
 841
 842         return 0;
 843 }
 844
 845 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)
 846 {
 847         struct hfi_sys_set_property_pkt *pkt;
 848         u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
 849
 850         pkt = (struct hfi_sys_set_property_pkt *)packet;
 851
 852         pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);
 853
 854         return venus_iface_cmdq_write(hdev, pkt, false);
 855 }
 856
 857 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)
 858 {
 859         struct hfi_sys_set_property_pkt *pkt;
 860         u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
 861
 862         pkt = (struct hfi_sys_set_property_pkt *)packet;
 863
 864         pkt_sys_coverage_config(pkt, mode);
 865
 866         return venus_iface_cmdq_write(hdev, pkt, false);
 867 }
 868
 869 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,
 870                                       bool enable)
 871 {
 872         struct hfi_sys_set_property_pkt *pkt;
 873         u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
 874
 875         if (!enable)
 876                 return 0;
 877
 878         pkt = (struct hfi_sys_set_property_pkt *)packet;
 879
 880         pkt_sys_idle_indicator(pkt, enable);
 881
 882         return venus_iface_cmdq_write(hdev, pkt, false);
 883 }
 884
 885 static int venus_sys_set_power_control(struct venus_hfi_device *hdev,
 886                                        bool enable)
 887 {
 888         struct hfi_sys_set_property_pkt *pkt;
 889         u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
 890
 891         pkt = (struct hfi_sys_set_property_pkt *)packet;
 892
 893         pkt_sys_power_control(pkt, enable);
 894
 895         return venus_iface_cmdq_write(hdev, pkt, false);
 896 }
 897
 898 static int venus_sys_set_ubwc_config(struct venus_hfi_device *hdev)
 899 {
 900         struct hfi_sys_set_property_pkt *pkt;
 901         u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
 902         const struct venus_resources *res = hdev->core->res;
 903         int ret;
 904
 905         pkt = (struct hfi_sys_set_property_pkt *)packet;
 906
 907         pkt_sys_ubwc_config(pkt, res->ubwc_conf);
 908
 909         ret = venus_iface_cmdq_write(hdev, pkt, false);
 910         if (ret)
 911                 return ret;
 912
 913         return 0;
 914 }
 915
 916 static int venus_get_queue_size(struct venus_hfi_device *hdev,
 917                                 unsigned int index)
 918 {
 919         struct hfi_queue_header *qhdr;
 920
 921         if (index >= IFACEQ_NUM)
 922                 return -EINVAL;
 923
 924         qhdr = hdev->queues[index].qhdr;
 925         if (!qhdr)
 926                 return -EINVAL;
 927
 928         return abs(qhdr->read_idx - qhdr->write_idx);
 929 }
 930
 931 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)
 932 {
 933         struct device *dev = hdev->core->dev;
 934         const struct venus_resources *res = hdev->core->res;
 935         int ret;
 936
 937         ret = venus_sys_set_debug(hdev, venus_fw_debug);
 938         if (ret)
 939                 dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);
 940
 941         /* HFI_PROPERTY_SYS_IDLE_INDICATOR is not supported beyond 8916 (HFI V1) */
 942         if (IS_V1(hdev->core)) {
 943                 ret = venus_sys_set_idle_message(hdev, false);
 944                 if (ret)
 945                         dev_warn(dev, "setting idle response ON failed (%d)\n", ret);
 946         }
 947
 948         ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);
 949         if (ret)
 950                 dev_warn(dev, "setting hw power collapse ON failed (%d)\n",
 951                          ret);
 952
 953         /* For specific venus core, it is mandatory to set the UBWC configuration */
 954         if (res->ubwc_conf) {
 955                 ret = venus_sys_set_ubwc_config(hdev);
 956                 if (ret)
 957                         dev_warn(dev, "setting ubwc config failed (%d)\n", ret);
 958         }
 959
 960         return ret;
 961 }
 962
 963 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync)
 964 {
 965         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
 966         struct hfi_session_pkt pkt;
 967
 968         pkt_session_cmd(&pkt, pkt_type, inst);
 969
 970         return venus_iface_cmdq_write(hdev, &pkt, sync);
 971 }
 972
 973 static void venus_flush_debug_queue(struct venus_hfi_device *hdev)
 974 {
 975         struct device *dev = hdev->core->dev;
 976         void *packet = hdev->dbg_buf;
 977
 978         while (!venus_iface_dbgq_read(hdev, packet)) {
 979                 struct hfi_msg_sys_coverage_pkt *pkt = packet;
 980
 981                 if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
 982                         struct hfi_msg_sys_debug_pkt *pkt = packet;
 983
 984                         dev_dbg(dev, VDBGFW "%s", pkt->msg_data);
 985                 }
 986         }
 987 }
 988
 989 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,
 990                                         bool wait)
 991 {
 992         unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);
 993         struct hfi_sys_pc_prep_pkt pkt;
 994         int ret;
 995
 996         init_completion(&hdev->pwr_collapse_prep);
 997
 998         pkt_sys_pc_prep(&pkt);
 999
1000         ret = venus_iface_cmdq_write(hdev, &pkt, false);
1001         if (ret)
1002                 return ret;
1003
1004         if (!wait)
1005                 return 0;
1006
1007         ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);
1008         if (!ret) {
1009                 venus_flush_debug_queue(hdev);
1010                 return -ETIMEDOUT;
1011         }
1012
1013         return 0;
1014 }
1015
1016 static int venus_are_queues_empty(struct venus_hfi_device *hdev)
1017 {
1018         int ret1, ret2;
1019
1020         ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);
1021         if (ret1 < 0)
1022                 return ret1;
1023
1024         ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);
1025         if (ret2 < 0)
1026                 return ret2;
1027
1028         if (!ret1 && !ret2)
1029                 return 1;
1030
1031         return 0;
1032 }
1033
1034 static void venus_sfr_print(struct venus_hfi_device *hdev)
1035 {
1036         struct device *dev = hdev->core->dev;
1037         struct hfi_sfr *sfr = hdev->sfr.kva;
1038         void *p;
1039
1040         if (!sfr)
1041                 return;
1042
1043         p = memchr(sfr->data, '\0', sfr->buf_size);
1044         /*
1045          * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates
1046          * that Venus is in the process of crashing.
1047          */
1048         if (!p)
1049                 sfr->data[sfr->buf_size - 1] = '\0';
1050
1051         dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);
1052 }
1053
1054 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,
1055                                         void *packet)
1056 {
1057         struct hfi_msg_event_notify_pkt *event_pkt = packet;
1058
1059         if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)
1060                 return;
1061
1062         venus_set_state(hdev, VENUS_STATE_DEINIT);
1063
1064         venus_sfr_print(hdev);
1065 }
1066
1067 static irqreturn_t venus_isr_thread(struct venus_core *core)
1068 {
1069         struct venus_hfi_device *hdev = to_hfi_priv(core);
1070         const struct venus_resources *res;
1071         void *pkt;
1072         u32 msg_ret;
1073
1074         if (!hdev)
1075                 return IRQ_NONE;
1076
1077         res = hdev->core->res;
1078         pkt = hdev->pkt_buf;
1079
1080
1081         while (!venus_iface_msgq_read(hdev, pkt)) {
1082                 msg_ret = hfi_process_msg_packet(core, pkt);
1083                 switch (msg_ret) {
1084                 case HFI_MSG_EVENT_NOTIFY:
1085                         venus_process_msg_sys_error(hdev, pkt);
1086                         break;
1087                 case HFI_MSG_SYS_INIT:
1088                         venus_hfi_core_set_resource(core, res->vmem_id,
1089                                                     res->vmem_size,
1090                                                     res->vmem_addr,
1091                                                     hdev);
1092                         break;
1093                 case HFI_MSG_SYS_RELEASE_RESOURCE:
1094                         complete(&hdev->release_resource);
1095                         break;
1096                 case HFI_MSG_SYS_PC_PREP:
1097                         complete(&hdev->pwr_collapse_prep);
1098                         break;
1099                 default:
1100                         break;
1101                 }
1102         }
1103
1104         venus_flush_debug_queue(hdev);
1105
1106         return IRQ_HANDLED;
1107 }
1108
1109 static irqreturn_t venus_isr(struct venus_core *core)
1110 {
1111         struct venus_hfi_device *hdev = to_hfi_priv(core);
1112         u32 status;
1113         void __iomem *cpu_cs_base;
1114         void __iomem *wrapper_base;
1115
1116         if (!hdev)
1117                 return IRQ_NONE;
1118
1119         cpu_cs_base = hdev->core->cpu_cs_base;
1120         wrapper_base = hdev->core->wrapper_base;
1121
1122         status = readl(wrapper_base + WRAPPER_INTR_STATUS);
1123         if (IS_IRIS2(core) || IS_IRIS2_1(core)) {
1124                 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1125                     status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 ||
1126                     status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1127                         hdev->irq_status = status;
1128         } else {
1129                 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1130                     status & WRAPPER_INTR_STATUS_A2HWD_MASK ||
1131                     status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1132                         hdev->irq_status = status;
1133         }
1134         writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR);
1135         if (!(IS_IRIS2(core) || IS_IRIS2_1(core)))
1136                 writel(status, wrapper_base + WRAPPER_INTR_CLEAR);
1137
1138         return IRQ_WAKE_THREAD;
1139 }
1140
1141 static int venus_core_init(struct venus_core *core)
1142 {
1143         struct venus_hfi_device *hdev = to_hfi_priv(core);
1144         struct device *dev = core->dev;
1145         struct hfi_sys_get_property_pkt version_pkt;
1146         struct hfi_sys_init_pkt pkt;
1147         int ret;
1148
1149         pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);
1150
1151         venus_set_state(hdev, VENUS_STATE_INIT);
1152
1153         ret = venus_iface_cmdq_write(hdev, &pkt, false);
1154         if (ret)
1155                 return ret;
1156
1157         pkt_sys_image_version(&version_pkt);
1158
1159         ret = venus_iface_cmdq_write(hdev, &version_pkt, false);
1160         if (ret)
1161                 dev_warn(dev, "failed to send image version pkt to fw\n");
1162
1163         ret = venus_sys_set_default_properties(hdev);
1164         if (ret)
1165                 return ret;
1166
1167         return 0;
1168 }
1169
1170 static int venus_core_deinit(struct venus_core *core)
1171 {
1172         struct venus_hfi_device *hdev = to_hfi_priv(core);
1173
1174         venus_set_state(hdev, VENUS_STATE_DEINIT);
1175         hdev->suspended = true;
1176         hdev->power_enabled = false;
1177
1178         return 0;
1179 }
1180
1181 static int venus_core_ping(struct venus_core *core, u32 cookie)
1182 {
1183         struct venus_hfi_device *hdev = to_hfi_priv(core);
1184         struct hfi_sys_ping_pkt pkt;
1185
1186         pkt_sys_ping(&pkt, cookie);
1187
1188         return venus_iface_cmdq_write(hdev, &pkt, false);
1189 }
1190
1191 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)
1192 {
1193         struct venus_hfi_device *hdev = to_hfi_priv(core);
1194         struct hfi_sys_test_ssr_pkt pkt;
1195         int ret;
1196
1197         ret = pkt_sys_ssr_cmd(&pkt, trigger_type);
1198         if (ret)
1199                 return ret;
1200
1201         return venus_iface_cmdq_write(hdev, &pkt, false);
1202 }
1203
1204 static int venus_session_init(struct venus_inst *inst, u32 session_type,
1205                               u32 codec)
1206 {
1207         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1208         struct hfi_session_init_pkt pkt;
1209         int ret;
1210
1211         ret = venus_sys_set_debug(hdev, venus_fw_debug);
1212         if (ret)
1213                 goto err;
1214
1215         ret = pkt_session_init(&pkt, inst, session_type, codec);
1216         if (ret)
1217                 goto err;
1218
1219         ret = venus_iface_cmdq_write(hdev, &pkt, true);
1220         if (ret)
1221                 goto err;
1222
1223         return 0;
1224
1225 err:
1226         venus_flush_debug_queue(hdev);
1227         return ret;
1228 }
1229
1230 static int venus_session_end(struct venus_inst *inst)
1231 {
1232         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1233         struct device *dev = hdev->core->dev;
1234
1235         if (venus_fw_coverage) {
1236                 if (venus_sys_set_coverage(hdev, venus_fw_coverage))
1237                         dev_warn(dev, "fw coverage msg ON failed\n");
1238         }
1239
1240         return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true);
1241 }
1242
1243 static int venus_session_abort(struct venus_inst *inst)
1244 {
1245         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1246
1247         venus_flush_debug_queue(hdev);
1248
1249         return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true);
1250 }
1251
1252 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)
1253 {
1254         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1255         struct hfi_session_flush_pkt pkt;
1256         int ret;
1257
1258         ret = pkt_session_flush(&pkt, inst, flush_mode);
1259         if (ret)
1260                 return ret;
1261
1262         return venus_iface_cmdq_write(hdev, &pkt, true);
1263 }
1264
1265 static int venus_session_start(struct venus_inst *inst)
1266 {
1267         return venus_session_cmd(inst, HFI_CMD_SESSION_START, true);
1268 }
1269
1270 static int venus_session_stop(struct venus_inst *inst)
1271 {
1272         return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true);
1273 }
1274
1275 static int venus_session_continue(struct venus_inst *inst)
1276 {
1277         return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false);
1278 }
1279
1280 static int venus_session_etb(struct venus_inst *inst,
1281                              struct hfi_frame_data *in_frame)
1282 {
1283         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1284         u32 session_type = inst->session_type;
1285         int ret;
1286
1287         if (session_type == VIDC_SESSION_TYPE_DEC) {
1288                 struct hfi_session_empty_buffer_compressed_pkt pkt;
1289
1290                 ret = pkt_session_etb_decoder(&pkt, inst, in_frame);
1291                 if (ret)
1292                         return ret;
1293
1294                 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1295         } else if (session_type == VIDC_SESSION_TYPE_ENC) {
1296                 struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;
1297
1298                 ret = pkt_session_etb_encoder(&pkt, inst, in_frame);
1299                 if (ret)
1300                         return ret;
1301
1302                 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1303         } else {
1304                 ret = -EINVAL;
1305         }
1306
1307         return ret;
1308 }
1309
1310 static int venus_session_ftb(struct venus_inst *inst,
1311                              struct hfi_frame_data *out_frame)
1312 {
1313         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1314         struct hfi_session_fill_buffer_pkt pkt;
1315         int ret;
1316
1317         ret = pkt_session_ftb(&pkt, inst, out_frame);
1318         if (ret)
1319                 return ret;
1320
1321         return venus_iface_cmdq_write(hdev, &pkt, false);
1322 }
1323
1324 static int venus_session_set_buffers(struct venus_inst *inst,
1325                                      struct hfi_buffer_desc *bd)
1326 {
1327         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1328         struct hfi_session_set_buffers_pkt *pkt;
1329         u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1330         int ret;
1331
1332         if (bd->buffer_type == HFI_BUFFER_INPUT)
1333                 return 0;
1334
1335         pkt = (struct hfi_session_set_buffers_pkt *)packet;
1336
1337         ret = pkt_session_set_buffers(pkt, inst, bd);
1338         if (ret)
1339                 return ret;
1340
1341         return venus_iface_cmdq_write(hdev, pkt, false);
1342 }
1343
1344 static int venus_session_unset_buffers(struct venus_inst *inst,
1345                                        struct hfi_buffer_desc *bd)
1346 {
1347         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1348         struct hfi_session_release_buffer_pkt *pkt;
1349         u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1350         int ret;
1351
1352         if (bd->buffer_type == HFI_BUFFER_INPUT)
1353                 return 0;
1354
1355         pkt = (struct hfi_session_release_buffer_pkt *)packet;
1356
1357         ret = pkt_session_unset_buffers(pkt, inst, bd);
1358         if (ret)
1359                 return ret;
1360
1361         return venus_iface_cmdq_write(hdev, pkt, true);
1362 }
1363
1364 static int venus_session_load_res(struct venus_inst *inst)
1365 {
1366         return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true);
1367 }
1368
1369 static int venus_session_release_res(struct venus_inst *inst)
1370 {
1371         return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true);
1372 }
1373
1374 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1375                                        u32 seq_hdr_len)
1376 {
1377         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1378         struct hfi_session_parse_sequence_header_pkt *pkt;
1379         u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1380         int ret;
1381
1382         pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;
1383
1384         ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);
1385         if (ret)
1386                 return ret;
1387
1388         ret = venus_iface_cmdq_write(hdev, pkt, false);
1389         if (ret)
1390                 return ret;
1391
1392         return 0;
1393 }
1394
1395 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1396                                      u32 seq_hdr_len)
1397 {
1398         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1399         struct hfi_session_get_sequence_header_pkt *pkt;
1400         u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1401         int ret;
1402
1403         pkt = (struct hfi_session_get_sequence_header_pkt *)packet;
1404
1405         ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);
1406         if (ret)
1407                 return ret;
1408
1409         return venus_iface_cmdq_write(hdev, pkt, false);
1410 }
1411
1412 static int venus_session_set_property(struct venus_inst *inst, u32 ptype,
1413                                       void *pdata)
1414 {
1415         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1416         struct hfi_session_set_property_pkt *pkt;
1417         u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1418         int ret;
1419
1420         pkt = (struct hfi_session_set_property_pkt *)packet;
1421
1422         ret = pkt_session_set_property(pkt, inst, ptype, pdata);
1423         if (ret == -ENOTSUPP)
1424                 return 0;
1425         if (ret)
1426                 return ret;
1427
1428         return venus_iface_cmdq_write(hdev, pkt, false);
1429 }
1430
1431 static int venus_session_get_property(struct venus_inst *inst, u32 ptype)
1432 {
1433         struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1434         struct hfi_session_get_property_pkt pkt;
1435         int ret;
1436
1437         ret = pkt_session_get_property(&pkt, inst, ptype);
1438         if (ret)
1439                 return ret;
1440
1441         return venus_iface_cmdq_write(hdev, &pkt, true);
1442 }
1443
1444 static int venus_resume(struct venus_core *core)
1445 {
1446         struct venus_hfi_device *hdev = to_hfi_priv(core);
1447         int ret = 0;
1448
1449         mutex_lock(&hdev->lock);
1450
1451         if (!hdev->suspended)
1452                 goto unlock;
1453
1454         ret = venus_power_on(hdev);
1455
1456 unlock:
1457         if (!ret)
1458                 hdev->suspended = false;
1459
1460         mutex_unlock(&hdev->lock);
1461
1462         return ret;
1463 }
1464
1465 static int venus_suspend_1xx(struct venus_core *core)
1466 {
1467         struct venus_hfi_device *hdev = to_hfi_priv(core);
1468         struct device *dev = core->dev;
1469         void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1470         u32 ctrl_status;
1471         int ret;
1472
1473         if (!hdev->power_enabled || hdev->suspended)
1474                 return 0;
1475
1476         mutex_lock(&hdev->lock);
1477         ret = venus_is_valid_state(hdev);
1478         mutex_unlock(&hdev->lock);
1479
1480         if (!ret) {
1481                 dev_err(dev, "bad state, cannot suspend\n");
1482                 return -EINVAL;
1483         }
1484
1485         ret = venus_prepare_power_collapse(hdev, true);
1486         if (ret) {
1487                 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1488                 return ret;
1489         }
1490
1491         mutex_lock(&hdev->lock);
1492
1493         if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {
1494                 mutex_unlock(&hdev->lock);
1495                 return -EINVAL;
1496         }
1497
1498         ret = venus_are_queues_empty(hdev);
1499         if (ret < 0 || !ret) {
1500                 mutex_unlock(&hdev->lock);
1501                 return -EINVAL;
1502         }
1503
1504         ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1505         if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {
1506                 mutex_unlock(&hdev->lock);
1507                 return -EINVAL;
1508         }
1509
1510         ret = venus_power_off(hdev);
1511         if (ret) {
1512                 mutex_unlock(&hdev->lock);
1513                 return ret;
1514         }
1515
1516         hdev->suspended = true;
1517
1518         mutex_unlock(&hdev->lock);
1519
1520         return 0;
1521 }
1522
1523 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)
1524 {
1525         void __iomem *wrapper_base = hdev->core->wrapper_base;
1526         void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1527         void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1528         u32 ctrl_status, cpu_status;
1529
1530         if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))
1531                 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1532         else
1533                 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1534         ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1535
1536         if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1537             ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1538                 return true;
1539
1540         return false;
1541 }
1542
1543 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)
1544 {
1545         void __iomem *wrapper_base = hdev->core->wrapper_base;
1546         void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1547         void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1548         u32 ctrl_status, cpu_status;
1549
1550         if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))
1551                 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1552         else
1553                 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1554         ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1555
1556         if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1557             ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1558                 return true;
1559
1560         return false;
1561 }
1562
1563 static int venus_suspend_3xx(struct venus_core *core)
1564 {
1565         struct venus_hfi_device *hdev = to_hfi_priv(core);
1566         struct device *dev = core->dev;
1567         void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1568         u32 ctrl_status;
1569         bool val;
1570         int ret;
1571
1572         if (!hdev->power_enabled || hdev->suspended)
1573                 return 0;
1574
1575         mutex_lock(&hdev->lock);
1576         ret = venus_is_valid_state(hdev);
1577         mutex_unlock(&hdev->lock);
1578
1579         if (!ret) {
1580                 dev_err(dev, "bad state, cannot suspend\n");
1581                 return -EINVAL;
1582         }
1583
1584         ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1585         if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1586                 goto power_off;
1587
1588         /*
1589          * Power collapse sequence for Venus 3xx and 4xx versions:
1590          * 1. Check for ARM9 and video core to be idle by checking WFI bit
1591          *    (bit 0) in CPU status register and by checking Idle (bit 30) in
1592          *    Control status register for video core.
1593          * 2. Send a command to prepare for power collapse.
1594          * 3. Check for WFI and PC_READY bits.
1595          */
1596         ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,
1597                                  1500, 100 * 1500);
1598         if (ret) {
1599                 dev_err(dev, "wait for cpu and video core idle fail (%d)\n", ret);
1600                 return ret;
1601         }
1602
1603         ret = venus_prepare_power_collapse(hdev, false);
1604         if (ret) {
1605                 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1606                 return ret;
1607         }
1608
1609         ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,
1610                                  1500, 100 * 1500);
1611         if (ret)
1612                 return ret;
1613
1614 power_off:
1615         mutex_lock(&hdev->lock);
1616
1617         ret = venus_power_off(hdev);
1618         if (ret) {
1619                 dev_err(dev, "venus_power_off (%d)\n", ret);
1620                 mutex_unlock(&hdev->lock);
1621                 return ret;
1622         }
1623
1624         hdev->suspended = true;
1625
1626         mutex_unlock(&hdev->lock);
1627
1628         return 0;
1629 }
1630
1631 static int venus_suspend(struct venus_core *core)
1632 {
1633         if (IS_V3(core) || IS_V4(core) || IS_V6(core))
1634                 return venus_suspend_3xx(core);
1635
1636         return venus_suspend_1xx(core);
1637 }
1638
1639 static const struct hfi_ops venus_hfi_ops = {
1640         .core_init                      = venus_core_init,
1641         .core_deinit                    = venus_core_deinit,
1642         .core_ping                      = venus_core_ping,
1643         .core_trigger_ssr               = venus_core_trigger_ssr,
1644
1645         .session_init                   = venus_session_init,
1646         .session_end                    = venus_session_end,
1647         .session_abort                  = venus_session_abort,
1648         .session_flush                  = venus_session_flush,
1649         .session_start                  = venus_session_start,
1650         .session_stop                   = venus_session_stop,
1651         .session_continue               = venus_session_continue,
1652         .session_etb                    = venus_session_etb,
1653         .session_ftb                    = venus_session_ftb,
1654         .session_set_buffers            = venus_session_set_buffers,
1655         .session_unset_buffers          = venus_session_unset_buffers,
1656         .session_load_res               = venus_session_load_res,
1657         .session_release_res            = venus_session_release_res,
1658         .session_parse_seq_hdr          = venus_session_parse_seq_hdr,
1659         .session_get_seq_hdr            = venus_session_get_seq_hdr,
1660         .session_set_property           = venus_session_set_property,
1661         .session_get_property           = venus_session_get_property,
1662
1663         .resume                         = venus_resume,
1664         .suspend                        = venus_suspend,
1665
1666         .isr                            = venus_isr,
1667         .isr_thread                     = venus_isr_thread,
1668 };
1669
1670 void venus_hfi_destroy(struct venus_core *core)
1671 {
1672         struct venus_hfi_device *hdev = to_hfi_priv(core);
1673
1674         core->priv = NULL;
1675         venus_interface_queues_release(hdev);
1676         mutex_destroy(&hdev->lock);
1677         kfree(hdev);
1678         core->ops = NULL;
1679 }
1680
1681 int venus_hfi_create(struct venus_core *core)
1682 {
1683         struct venus_hfi_device *hdev;
1684         int ret;
1685
1686         hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
1687         if (!hdev)
1688                 return -ENOMEM;
1689
1690         mutex_init(&hdev->lock);
1691
1692         hdev->core = core;
1693         hdev->suspended = true;
1694         core->priv = hdev;
1695         core->ops = &venus_hfi_ops;
1696
1697         ret = venus_interface_queues_init(hdev);
1698         if (ret)
1699                 goto err_kfree;
1700
1701         return 0;
1702
1703 err_kfree:
1704         kfree(hdev);
1705         core->priv = NULL;
1706         core->ops = NULL;
1707         return ret;
1708 }
1709
1710 void venus_hfi_queues_reinit(struct venus_core *core)
1711 {
1712         struct venus_hfi_device *hdev = to_hfi_priv(core);
1713         struct hfi_queue_table_header *tbl_hdr;
1714         struct iface_queue *queue;
1715         struct hfi_sfr *sfr;
1716         unsigned int i;
1717
1718         mutex_lock(&hdev->lock);
1719
1720         for (i = 0; i < IFACEQ_NUM; i++) {
1721                 queue = &hdev->queues[i];
1722                 queue->qhdr =
1723                         IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
1724
1725                 venus_set_qhdr_defaults(queue->qhdr);
1726
1727                 queue->qhdr->start_addr = queue->qmem.da;
1728
1729                 if (i == IFACEQ_CMD_IDX)
1730                         queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
1731                 else if (i == IFACEQ_MSG_IDX)
1732                         queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
1733                 else if (i == IFACEQ_DBG_IDX)
1734                         queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
1735         }
1736
1737         tbl_hdr = hdev->ifaceq_table.kva;
1738         tbl_hdr->version = 0;
1739         tbl_hdr->size = IFACEQ_TABLE_SIZE;
1740         tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
1741         tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
1742         tbl_hdr->num_q = IFACEQ_NUM;
1743         tbl_hdr->num_active_q = IFACEQ_NUM;
1744
1745         /*
1746          * Set receive request to zero on debug queue as there is no
1747          * need of interrupt from video hardware for debug messages
1748          */
1749         queue = &hdev->queues[IFACEQ_DBG_IDX];
1750         queue->qhdr->rx_req = 0;
1751
1752         sfr = hdev->sfr.kva;
1753         sfr->buf_size = ALIGNED_SFR_SIZE;
1754
1755         /* ensure table and queue header structs are settled in memory */
1756         wmb();
1757
1758         mutex_unlock(&hdev->lock);
1759 }