drivers/net/wwan/t7xx/t7xx_hif_dpmaif_rx.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2021, MediaTek Inc.
   4  * Copyright (c) 2021-2022, Intel Corporation.
   5  *
   6  * Authors:
   7  *  Amir Hanania <amir.hanania@intel.com>
   8  *  Haijun Liu <haijun.liu@mediatek.com>
   9  *  Eliot Lee <eliot.lee@intel.com>
  10  *  Moises Veleta <moises.veleta@intel.com>
  11  *  Ricardo Martinez <ricardo.martinez@linux.intel.com>
  12  *
  13  * Contributors:
  14  *  Andy Shevchenko <andriy.shevchenko@linux.intel.com>
  15  *  Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com>
  16  *  Sreehari Kancharla <sreehari.kancharla@intel.com>
  17  */
  18
  19 #include <linux/atomic.h>
  20 #include <linux/bitfield.h>
  21 #include <linux/bitops.h>
  22 #include <linux/device.h>
  23 #include <linux/dma-direction.h>
  24 #include <linux/dma-mapping.h>
  25 #include <linux/gfp.h>
  26 #include <linux/err.h>
  27 #include <linux/iopoll.h>
  28 #include <linux/jiffies.h>
  29 #include <linux/kernel.h>
  30 #include <linux/kthread.h>
  31 #include <linux/list.h>
  32 #include <linux/minmax.h>
  33 #include <linux/mm.h>
  34 #include <linux/netdevice.h>
  35 #include <linux/pm_runtime.h>
  36 #include <linux/sched.h>
  37 #include <linux/skbuff.h>
  38 #include <linux/slab.h>
  39 #include <linux/spinlock.h>
  40 #include <linux/string.h>
  41 #include <linux/types.h>
  42 #include <linux/wait.h>
  43 #include <linux/workqueue.h>
  44
  45 #include "t7xx_dpmaif.h"
  46 #include "t7xx_hif_dpmaif.h"
  47 #include "t7xx_hif_dpmaif_rx.h"
  48 #include "t7xx_netdev.h"
  49 #include "t7xx_pci.h"
  50
  51 #define DPMAIF_BAT_COUNT                8192
  52 #define DPMAIF_FRG_COUNT                4814
  53 #define DPMAIF_PIT_COUNT                (DPMAIF_BAT_COUNT * 2)
  54
  55 #define DPMAIF_BAT_CNT_THRESHOLD        30
  56 #define DPMAIF_PIT_CNT_THRESHOLD        60
  57 #define DPMAIF_RX_PUSH_THRESHOLD_MASK   GENMASK(2, 0)
  58 #define DPMAIF_NOTIFY_RELEASE_COUNT     128
  59 #define DPMAIF_POLL_PIT_TIME_US         20
  60 #define DPMAIF_POLL_PIT_MAX_TIME_US     2000
  61 #define DPMAIF_WQ_TIME_LIMIT_MS         2
  62 #define DPMAIF_CS_RESULT_PASS           0
  63
  64 /* Packet type */
  65 #define DES_PT_PD                       0
  66 #define DES_PT_MSG                      1
  67 /* Buffer type */
  68 #define PKT_BUF_FRAG                    1
  69
  70 static unsigned int t7xx_normal_pit_bid(const struct dpmaif_pit *pit_info)
  71 {
  72         u32 value;
  73
  74         value = FIELD_GET(PD_PIT_H_BID, le32_to_cpu(pit_info->pd.footer));
  75         value <<= 13;
  76         value += FIELD_GET(PD_PIT_BUFFER_ID, le32_to_cpu(pit_info->header));
  77         return value;
  78 }
  79
  80 static int t7xx_dpmaif_update_bat_wr_idx(struct dpmaif_ctrl *dpmaif_ctrl,
  81                                          const unsigned int q_num, const unsigned int bat_cnt)
  82 {
  83         struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[q_num];
  84         struct dpmaif_bat_request *bat_req = rxq->bat_req;
  85         unsigned int old_rl_idx, new_wr_idx, old_wr_idx;
  86
  87         if (!rxq->que_started) {
  88                 dev_err(dpmaif_ctrl->dev, "RX queue %d has not been started\n", rxq->index);
  89                 return -EINVAL;
  90         }
  91
  92         old_rl_idx = bat_req->bat_release_rd_idx;
  93         old_wr_idx = bat_req->bat_wr_idx;
  94         new_wr_idx = old_wr_idx + bat_cnt;
  95
  96         if (old_rl_idx > old_wr_idx && new_wr_idx >= old_rl_idx)
  97                 goto err_flow;
  98
  99         if (new_wr_idx >= bat_req->bat_size_cnt) {
 100                 new_wr_idx -= bat_req->bat_size_cnt;
 101                 if (new_wr_idx >= old_rl_idx)
 102                         goto err_flow;
 103         }
 104
 105         bat_req->bat_wr_idx = new_wr_idx;
 106         return 0;
 107
 108 err_flow:
 109         dev_err(dpmaif_ctrl->dev, "RX BAT flow check fail\n");
 110         return -EINVAL;
 111 }
 112
 113 static bool t7xx_alloc_and_map_skb_info(const struct dpmaif_ctrl *dpmaif_ctrl,
 114                                         const unsigned int size, struct dpmaif_bat_skb *cur_skb)
 115 {
 116         dma_addr_t data_bus_addr;
 117         struct sk_buff *skb;
 118
 119         skb = __dev_alloc_skb(size, GFP_KERNEL);
 120         if (!skb)
 121                 return false;
 122
 123         data_bus_addr = dma_map_single(dpmaif_ctrl->dev, skb->data, size, DMA_FROM_DEVICE);
 124         if (dma_mapping_error(dpmaif_ctrl->dev, data_bus_addr)) {
 125                 dev_err_ratelimited(dpmaif_ctrl->dev, "DMA mapping error\n");
 126                 dev_kfree_skb_any(skb);
 127                 return false;
 128         }
 129
 130         cur_skb->skb = skb;
 131         cur_skb->data_bus_addr = data_bus_addr;
 132         cur_skb->data_len = size;
 133
 134         return true;
 135 }
 136
 137 static void t7xx_unmap_bat_skb(struct device *dev, struct dpmaif_bat_skb *bat_skb_base,
 138                                unsigned int index)
 139 {
 140         struct dpmaif_bat_skb *bat_skb = bat_skb_base + index;
 141
 142         if (bat_skb->skb) {
 143                 dma_unmap_single(dev, bat_skb->data_bus_addr, bat_skb->data_len, DMA_FROM_DEVICE);
 144                 dev_kfree_skb(bat_skb->skb);
 145                 bat_skb->skb = NULL;
 146         }
 147 }
 148
 149 /**
 150  * t7xx_dpmaif_rx_buf_alloc() - Allocate buffers for the BAT ring.
 151  * @dpmaif_ctrl: Pointer to DPMAIF context structure.
 152  * @bat_req: Pointer to BAT request structure.
 153  * @q_num: Queue number.
 154  * @buf_cnt: Number of buffers to allocate.
 155  * @initial: Indicates if the ring is being populated for the first time.
 156  *
 157  * Allocate skb and store the start address of the data buffer into the BAT ring.
 158  * If this is not the initial call, notify the HW about the new entries.
 159  *
 160  * Return:
 161  * * 0          - Success.
 162  * * -ERROR     - Error code.
 163  */
 164 int t7xx_dpmaif_rx_buf_alloc(struct dpmaif_ctrl *dpmaif_ctrl,
 165                              const struct dpmaif_bat_request *bat_req,
 166                              const unsigned int q_num, const unsigned int buf_cnt,
 167                              const bool initial)
 168 {
 169         unsigned int i, bat_cnt, bat_max_cnt, bat_start_idx;
 170         int ret;
 171
 172         if (!buf_cnt || buf_cnt > bat_req->bat_size_cnt)
 173                 return -EINVAL;
 174
 175         /* Check BAT buffer space */
 176         bat_max_cnt = bat_req->bat_size_cnt;
 177
 178         bat_cnt = t7xx_ring_buf_rd_wr_count(bat_max_cnt, bat_req->bat_release_rd_idx,
 179                                             bat_req->bat_wr_idx, DPMAIF_WRITE);
 180         if (buf_cnt > bat_cnt)
 181                 return -ENOMEM;
 182
 183         bat_start_idx = bat_req->bat_wr_idx;
 184
 185         for (i = 0; i < buf_cnt; i++) {
 186                 unsigned int cur_bat_idx = bat_start_idx + i;
 187                 struct dpmaif_bat_skb *cur_skb;
 188                 struct dpmaif_bat *cur_bat;
 189
 190                 if (cur_bat_idx >= bat_max_cnt)
 191                         cur_bat_idx -= bat_max_cnt;
 192
 193                 cur_skb = (struct dpmaif_bat_skb *)bat_req->bat_skb + cur_bat_idx;
 194                 if (!cur_skb->skb &&
 195                     !t7xx_alloc_and_map_skb_info(dpmaif_ctrl, bat_req->pkt_buf_sz, cur_skb))
 196                         break;
 197
 198                 cur_bat = (struct dpmaif_bat *)bat_req->bat_base + cur_bat_idx;
 199                 cur_bat->buffer_addr_ext = upper_32_bits(cur_skb->data_bus_addr);
 200                 cur_bat->p_buffer_addr = lower_32_bits(cur_skb->data_bus_addr);
 201         }
 202
 203         if (!i)
 204                 return -ENOMEM;
 205
 206         ret = t7xx_dpmaif_update_bat_wr_idx(dpmaif_ctrl, q_num, i);
 207         if (ret)
 208                 goto err_unmap_skbs;
 209
 210         if (!initial) {
 211                 unsigned int hw_wr_idx;
 212
 213                 ret = t7xx_dpmaif_dl_snd_hw_bat_cnt(&dpmaif_ctrl->hw_info, i);
 214                 if (ret)
 215                         goto err_unmap_skbs;
 216
 217                 hw_wr_idx = t7xx_dpmaif_dl_get_bat_wr_idx(&dpmaif_ctrl->hw_info,
 218                                                           DPF_RX_QNO_DFT);
 219                 if (hw_wr_idx != bat_req->bat_wr_idx) {
 220                         ret = -EFAULT;
 221                         dev_err(dpmaif_ctrl->dev, "Write index mismatch in RX ring\n");
 222                         goto err_unmap_skbs;
 223                 }
 224         }
 225
 226         return 0;
 227
 228 err_unmap_skbs:
 229         while (--i > 0)
 230                 t7xx_unmap_bat_skb(dpmaif_ctrl->dev, bat_req->bat_skb, i);
 231
 232         return ret;
 233 }
 234
 235 static int t7xx_dpmaifq_release_pit_entry(struct dpmaif_rx_queue *rxq,
 236                                           const unsigned int rel_entry_num)
 237 {
 238         struct dpmaif_hw_info *hw_info = &rxq->dpmaif_ctrl->hw_info;
 239         unsigned int old_rel_idx, new_rel_idx, hw_wr_idx;
 240         int ret;
 241
 242         if (!rxq->que_started)
 243                 return 0;
 244
 245         if (rel_entry_num >= rxq->pit_size_cnt) {
 246                 dev_err(rxq->dpmaif_ctrl->dev, "Invalid PIT release index\n");
 247                 return -EINVAL;
 248         }
 249
 250         old_rel_idx = rxq->pit_release_rd_idx;
 251         new_rel_idx = old_rel_idx + rel_entry_num;
 252         hw_wr_idx = rxq->pit_wr_idx;
 253         if (hw_wr_idx < old_rel_idx && new_rel_idx >= rxq->pit_size_cnt)
 254                 new_rel_idx -= rxq->pit_size_cnt;
 255
 256         ret = t7xx_dpmaif_dlq_add_pit_remain_cnt(hw_info, rxq->index, rel_entry_num);
 257         if (ret) {
 258                 dev_err(rxq->dpmaif_ctrl->dev, "PIT release failure: %d\n", ret);
 259                 return ret;
 260         }
 261
 262         rxq->pit_release_rd_idx = new_rel_idx;
 263         return 0;
 264 }
 265
 266 static void t7xx_dpmaif_set_bat_mask(struct dpmaif_bat_request *bat_req, unsigned int idx)
 267 {
 268         unsigned long flags;
 269
 270         spin_lock_irqsave(&bat_req->mask_lock, flags);
 271         set_bit(idx, bat_req->bat_bitmap);
 272         spin_unlock_irqrestore(&bat_req->mask_lock, flags);
 273 }
 274
 275 static int t7xx_frag_bat_cur_bid_check(struct dpmaif_rx_queue *rxq,
 276                                        const unsigned int cur_bid)
 277 {
 278         struct dpmaif_bat_request *bat_frag = rxq->bat_frag;
 279         struct dpmaif_bat_page *bat_page;
 280
 281         if (cur_bid >= DPMAIF_FRG_COUNT)
 282                 return -EINVAL;
 283
 284         bat_page = bat_frag->bat_skb + cur_bid;
 285         if (!bat_page->page)
 286                 return -EINVAL;
 287
 288         return 0;
 289 }
 290
 291 static void t7xx_unmap_bat_page(struct device *dev, struct dpmaif_bat_page *bat_page_base,
 292                                 unsigned int index)
 293 {
 294         struct dpmaif_bat_page *bat_page = bat_page_base + index;
 295
 296         if (bat_page->page) {
 297                 dma_unmap_page(dev, bat_page->data_bus_addr, bat_page->data_len, DMA_FROM_DEVICE);
 298                 put_page(bat_page->page);
 299                 bat_page->page = NULL;
 300         }
 301 }
 302
 303 /**
 304  * t7xx_dpmaif_rx_frag_alloc() - Allocates buffers for the Fragment BAT ring.
 305  * @dpmaif_ctrl: Pointer to DPMAIF context structure.
 306  * @bat_req: Pointer to BAT request structure.
 307  * @buf_cnt: Number of buffers to allocate.
 308  * @initial: Indicates if the ring is being populated for the first time.
 309  *
 310  * Fragment BAT is used when the received packet does not fit in a normal BAT entry.
 311  * This function allocates a page fragment and stores the start address of the page
 312  * into the Fragment BAT ring.
 313  * If this is not the initial call, notify the HW about the new entries.
 314  *
 315  * Return:
 316  * * 0          - Success.
 317  * * -ERROR     - Error code.
 318  */
 319 int t7xx_dpmaif_rx_frag_alloc(struct dpmaif_ctrl *dpmaif_ctrl, struct dpmaif_bat_request *bat_req,
 320                               const unsigned int buf_cnt, const bool initial)
 321 {
 322         unsigned int buf_space, cur_bat_idx = bat_req->bat_wr_idx;
 323         struct dpmaif_bat_page *bat_skb = bat_req->bat_skb;
 324         int ret = 0, i;
 325
 326         if (!buf_cnt || buf_cnt > bat_req->bat_size_cnt)
 327                 return -EINVAL;
 328
 329         buf_space = t7xx_ring_buf_rd_wr_count(bat_req->bat_size_cnt,
 330                                               bat_req->bat_release_rd_idx, bat_req->bat_wr_idx,
 331                                               DPMAIF_WRITE);
 332         if (buf_cnt > buf_space) {
 333                 dev_err(dpmaif_ctrl->dev,
 334                         "Requested more buffers than the space available in RX frag ring\n");
 335                 return -EINVAL;
 336         }
 337
 338         for (i = 0; i < buf_cnt; i++) {
 339                 struct dpmaif_bat_page *cur_page = bat_skb + cur_bat_idx;
 340                 struct dpmaif_bat *cur_bat;
 341                 dma_addr_t data_base_addr;
 342
 343                 if (!cur_page->page) {
 344                         unsigned long offset;
 345                         struct page *page;
 346                         void *data;
 347
 348                         data = netdev_alloc_frag(bat_req->pkt_buf_sz);
 349                         if (!data)
 350                                 break;
 351
 352                         page = virt_to_head_page(data);
 353                         offset = data - page_address(page);
 354
 355                         data_base_addr = dma_map_page(dpmaif_ctrl->dev, page, offset,
 356                                                       bat_req->pkt_buf_sz, DMA_FROM_DEVICE);
 357                         if (dma_mapping_error(dpmaif_ctrl->dev, data_base_addr)) {
 358                                 put_page(virt_to_head_page(data));
 359                                 dev_err(dpmaif_ctrl->dev, "DMA mapping fail\n");
 360                                 break;
 361                         }
 362
 363                         cur_page->page = page;
 364                         cur_page->data_bus_addr = data_base_addr;
 365                         cur_page->offset = offset;
 366                         cur_page->data_len = bat_req->pkt_buf_sz;
 367                 }
 368
 369                 data_base_addr = cur_page->data_bus_addr;
 370                 cur_bat = (struct dpmaif_bat *)bat_req->bat_base + cur_bat_idx;
 371                 cur_bat->buffer_addr_ext = upper_32_bits(data_base_addr);
 372                 cur_bat->p_buffer_addr = lower_32_bits(data_base_addr);
 373                 cur_bat_idx = t7xx_ring_buf_get_next_wr_idx(bat_req->bat_size_cnt, cur_bat_idx);
 374         }
 375
 376         bat_req->bat_wr_idx = cur_bat_idx;
 377
 378         if (!initial)
 379                 t7xx_dpmaif_dl_snd_hw_frg_cnt(&dpmaif_ctrl->hw_info, i);
 380
 381         if (i < buf_cnt) {
 382                 ret = -ENOMEM;
 383                 if (initial) {
 384                         while (--i > 0)
 385                                 t7xx_unmap_bat_page(dpmaif_ctrl->dev, bat_req->bat_skb, i);
 386                 }
 387         }
 388
 389         return ret;
 390 }
 391
 392 static int t7xx_dpmaif_set_frag_to_skb(const struct dpmaif_rx_queue *rxq,
 393                                        const struct dpmaif_pit *pkt_info,
 394                                        struct sk_buff *skb)
 395 {
 396         unsigned long long data_bus_addr, data_base_addr;
 397         struct device *dev = rxq->dpmaif_ctrl->dev;
 398         struct dpmaif_bat_page *page_info;
 399         unsigned int data_len;
 400         int data_offset;
 401
 402         page_info = rxq->bat_frag->bat_skb;
 403         page_info += t7xx_normal_pit_bid(pkt_info);
 404         dma_unmap_page(dev, page_info->data_bus_addr, page_info->data_len, DMA_FROM_DEVICE);
 405
 406         if (!page_info->page)
 407                 return -EINVAL;
 408
 409         data_bus_addr = le32_to_cpu(pkt_info->pd.data_addr_h);
 410         data_bus_addr = (data_bus_addr << 32) + le32_to_cpu(pkt_info->pd.data_addr_l);
 411         data_base_addr = page_info->data_bus_addr;
 412         data_offset = data_bus_addr - data_base_addr;
 413         data_offset += page_info->offset;
 414         data_len = FIELD_GET(PD_PIT_DATA_LEN, le32_to_cpu(pkt_info->header));
 415         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page_info->page,
 416                         data_offset, data_len, page_info->data_len);
 417
 418         page_info->page = NULL;
 419         page_info->offset = 0;
 420         page_info->data_len = 0;
 421         return 0;
 422 }
 423
 424 static int t7xx_dpmaif_get_frag(struct dpmaif_rx_queue *rxq,
 425                                 const struct dpmaif_pit *pkt_info,
 426                                 const struct dpmaif_cur_rx_skb_info *skb_info)
 427 {
 428         unsigned int cur_bid = t7xx_normal_pit_bid(pkt_info);
 429         int ret;
 430
 431         ret = t7xx_frag_bat_cur_bid_check(rxq, cur_bid);
 432         if (ret < 0)
 433                 return ret;
 434
 435         ret = t7xx_dpmaif_set_frag_to_skb(rxq, pkt_info, skb_info->cur_skb);
 436         if (ret < 0) {
 437                 dev_err(rxq->dpmaif_ctrl->dev, "Failed to set frag data to skb: %d\n", ret);
 438                 return ret;
 439         }
 440
 441         t7xx_dpmaif_set_bat_mask(rxq->bat_frag, cur_bid);
 442         return 0;
 443 }
 444
 445 static int t7xx_bat_cur_bid_check(struct dpmaif_rx_queue *rxq, const unsigned int cur_bid)
 446 {
 447         struct dpmaif_bat_skb *bat_skb = rxq->bat_req->bat_skb;
 448
 449         bat_skb += cur_bid;
 450         if (cur_bid >= DPMAIF_BAT_COUNT || !bat_skb->skb)
 451                 return -EINVAL;
 452
 453         return 0;
 454 }
 455
 456 static int t7xx_dpmaif_read_pit_seq(const struct dpmaif_pit *pit)
 457 {
 458         return FIELD_GET(PD_PIT_PIT_SEQ, le32_to_cpu(pit->pd.footer));
 459 }
 460
 461 static int t7xx_dpmaif_check_pit_seq(struct dpmaif_rx_queue *rxq,
 462                                      const struct dpmaif_pit *pit)
 463 {
 464         unsigned int cur_pit_seq, expect_pit_seq = rxq->expect_pit_seq;
 465
 466         if (read_poll_timeout_atomic(t7xx_dpmaif_read_pit_seq, cur_pit_seq,
 467                                      cur_pit_seq == expect_pit_seq, DPMAIF_POLL_PIT_TIME_US,
 468                                      DPMAIF_POLL_PIT_MAX_TIME_US, false, pit))
 469                 return -EFAULT;
 470
 471         rxq->expect_pit_seq++;
 472         if (rxq->expect_pit_seq >= DPMAIF_DL_PIT_SEQ_VALUE)
 473                 rxq->expect_pit_seq = 0;
 474
 475         return 0;
 476 }
 477
 478 static unsigned int t7xx_dpmaif_avail_pkt_bat_cnt(struct dpmaif_bat_request *bat_req)
 479 {
 480         unsigned int zero_index;
 481         unsigned long flags;
 482
 483         spin_lock_irqsave(&bat_req->mask_lock, flags);
 484
 485         zero_index = find_next_zero_bit(bat_req->bat_bitmap, bat_req->bat_size_cnt,
 486                                         bat_req->bat_release_rd_idx);
 487
 488         if (zero_index < bat_req->bat_size_cnt) {
 489                 spin_unlock_irqrestore(&bat_req->mask_lock, flags);
 490                 return zero_index - bat_req->bat_release_rd_idx;
 491         }
 492
 493         /* limiting the search till bat_release_rd_idx */
 494         zero_index = find_first_zero_bit(bat_req->bat_bitmap, bat_req->bat_release_rd_idx);
 495         spin_unlock_irqrestore(&bat_req->mask_lock, flags);
 496         return bat_req->bat_size_cnt - bat_req->bat_release_rd_idx + zero_index;
 497 }
 498
 499 static int t7xx_dpmaif_release_bat_entry(const struct dpmaif_rx_queue *rxq,
 500                                          const unsigned int rel_entry_num,
 501                                          const enum bat_type buf_type)
 502 {
 503         struct dpmaif_hw_info *hw_info = &rxq->dpmaif_ctrl->hw_info;
 504         unsigned int old_rel_idx, new_rel_idx, hw_rd_idx, i;
 505         struct dpmaif_bat_request *bat;
 506         unsigned long flags;
 507
 508         if (!rxq->que_started || !rel_entry_num)
 509                 return -EINVAL;
 510
 511         if (buf_type == BAT_TYPE_FRAG) {
 512                 bat = rxq->bat_frag;
 513                 hw_rd_idx = t7xx_dpmaif_dl_get_frg_rd_idx(hw_info, rxq->index);
 514         } else {
 515                 bat = rxq->bat_req;
 516                 hw_rd_idx = t7xx_dpmaif_dl_get_bat_rd_idx(hw_info, rxq->index);
 517         }
 518
 519         if (rel_entry_num >= bat->bat_size_cnt)
 520                 return -EINVAL;
 521
 522         old_rel_idx = bat->bat_release_rd_idx;
 523         new_rel_idx = old_rel_idx + rel_entry_num;
 524
 525         /* Do not need to release if the queue is empty */
 526         if (bat->bat_wr_idx == old_rel_idx)
 527                 return 0;
 528
 529         if (hw_rd_idx >= old_rel_idx) {
 530                 if (new_rel_idx > hw_rd_idx)
 531                         return -EINVAL;
 532         }
 533
 534         if (new_rel_idx >= bat->bat_size_cnt) {
 535                 new_rel_idx -= bat->bat_size_cnt;
 536                 if (new_rel_idx > hw_rd_idx)
 537                         return -EINVAL;
 538         }
 539
 540         spin_lock_irqsave(&bat->mask_lock, flags);
 541         for (i = 0; i < rel_entry_num; i++) {
 542                 unsigned int index = bat->bat_release_rd_idx + i;
 543
 544                 if (index >= bat->bat_size_cnt)
 545                         index -= bat->bat_size_cnt;
 546
 547                 clear_bit(index, bat->bat_bitmap);
 548         }
 549         spin_unlock_irqrestore(&bat->mask_lock, flags);
 550
 551         bat->bat_release_rd_idx = new_rel_idx;
 552         return rel_entry_num;
 553 }
 554
 555 static int t7xx_dpmaif_pit_release_and_add(struct dpmaif_rx_queue *rxq)
 556 {
 557         int ret;
 558
 559         if (rxq->pit_remain_release_cnt < DPMAIF_PIT_CNT_THRESHOLD)
 560                 return 0;
 561
 562         ret = t7xx_dpmaifq_release_pit_entry(rxq, rxq->pit_remain_release_cnt);
 563         if (ret)
 564                 return ret;
 565
 566         rxq->pit_remain_release_cnt = 0;
 567         return 0;
 568 }
 569
 570 static int t7xx_dpmaif_bat_release_and_add(const struct dpmaif_rx_queue *rxq)
 571 {
 572         unsigned int bid_cnt;
 573         int ret;
 574
 575         bid_cnt = t7xx_dpmaif_avail_pkt_bat_cnt(rxq->bat_req);
 576         if (bid_cnt < DPMAIF_BAT_CNT_THRESHOLD)
 577                 return 0;
 578
 579         ret = t7xx_dpmaif_release_bat_entry(rxq, bid_cnt, BAT_TYPE_NORMAL);
 580         if (ret <= 0) {
 581                 dev_err(rxq->dpmaif_ctrl->dev, "Release PKT BAT failed: %d\n", ret);
 582                 return ret;
 583         }
 584
 585         ret = t7xx_dpmaif_rx_buf_alloc(rxq->dpmaif_ctrl, rxq->bat_req, rxq->index, bid_cnt, false);
 586         if (ret < 0)
 587                 dev_err(rxq->dpmaif_ctrl->dev, "Allocate new RX buffer failed: %d\n", ret);
 588
 589         return ret;
 590 }
 591
 592 static int t7xx_dpmaif_frag_bat_release_and_add(const struct dpmaif_rx_queue *rxq)
 593 {
 594         unsigned int bid_cnt;
 595         int ret;
 596
 597         bid_cnt = t7xx_dpmaif_avail_pkt_bat_cnt(rxq->bat_frag);
 598         if (bid_cnt < DPMAIF_BAT_CNT_THRESHOLD)
 599                 return 0;
 600
 601         ret = t7xx_dpmaif_release_bat_entry(rxq, bid_cnt, BAT_TYPE_FRAG);
 602         if (ret <= 0) {
 603                 dev_err(rxq->dpmaif_ctrl->dev, "Release BAT entry failed: %d\n", ret);
 604                 return ret;
 605         }
 606
 607         return t7xx_dpmaif_rx_frag_alloc(rxq->dpmaif_ctrl, rxq->bat_frag, bid_cnt, false);
 608 }
 609
 610 static void t7xx_dpmaif_parse_msg_pit(const struct dpmaif_rx_queue *rxq,
 611                                       const struct dpmaif_pit *msg_pit,
 612                                       struct dpmaif_cur_rx_skb_info *skb_info)
 613 {
 614         int header = le32_to_cpu(msg_pit->header);
 615
 616         skb_info->cur_chn_idx = FIELD_GET(MSG_PIT_CHANNEL_ID, header);
 617         skb_info->check_sum = FIELD_GET(MSG_PIT_CHECKSUM, header);
 618         skb_info->pit_dp = FIELD_GET(MSG_PIT_DP, header);
 619         skb_info->pkt_type = FIELD_GET(MSG_PIT_IP, le32_to_cpu(msg_pit->msg.params_3));
 620 }
 621
 622 static int t7xx_dpmaif_set_data_to_skb(const struct dpmaif_rx_queue *rxq,
 623                                        const struct dpmaif_pit *pkt_info,
 624                                        struct dpmaif_cur_rx_skb_info *skb_info)
 625 {
 626         unsigned long long data_bus_addr, data_base_addr;
 627         struct device *dev = rxq->dpmaif_ctrl->dev;
 628         struct dpmaif_bat_skb *bat_skb;
 629         unsigned int data_len;
 630         struct sk_buff *skb;
 631         int data_offset;
 632
 633         bat_skb = rxq->bat_req->bat_skb;
 634         bat_skb += t7xx_normal_pit_bid(pkt_info);
 635         dma_unmap_single(dev, bat_skb->data_bus_addr, bat_skb->data_len, DMA_FROM_DEVICE);
 636
 637         data_bus_addr = le32_to_cpu(pkt_info->pd.data_addr_h);
 638         data_bus_addr = (data_bus_addr << 32) + le32_to_cpu(pkt_info->pd.data_addr_l);
 639         data_base_addr = bat_skb->data_bus_addr;
 640         data_offset = data_bus_addr - data_base_addr;
 641         data_len = FIELD_GET(PD_PIT_DATA_LEN, le32_to_cpu(pkt_info->header));
 642         skb = bat_skb->skb;
 643         skb->len = 0;
 644         skb_reset_tail_pointer(skb);
 645         skb_reserve(skb, data_offset);
 646
 647         if (skb->tail + data_len > skb->end) {
 648                 dev_err(dev, "No buffer space available\n");
 649                 return -ENOBUFS;
 650         }
 651
 652         skb_put(skb, data_len);
 653         skb_info->cur_skb = skb;
 654         bat_skb->skb = NULL;
 655         return 0;
 656 }
 657
 658 static int t7xx_dpmaif_get_rx_pkt(struct dpmaif_rx_queue *rxq,
 659                                   const struct dpmaif_pit *pkt_info,
 660                                   struct dpmaif_cur_rx_skb_info *skb_info)
 661 {
 662         unsigned int cur_bid = t7xx_normal_pit_bid(pkt_info);
 663         int ret;
 664
 665         ret = t7xx_bat_cur_bid_check(rxq, cur_bid);
 666         if (ret < 0)
 667                 return ret;
 668
 669         ret = t7xx_dpmaif_set_data_to_skb(rxq, pkt_info, skb_info);
 670         if (ret < 0) {
 671                 dev_err(rxq->dpmaif_ctrl->dev, "RX set data to skb failed: %d\n", ret);
 672                 return ret;
 673         }
 674
 675         t7xx_dpmaif_set_bat_mask(rxq->bat_req, cur_bid);
 676         return 0;
 677 }
 678
 679 static int t7xx_dpmaifq_rx_notify_hw(struct dpmaif_rx_queue *rxq)
 680 {
 681         struct dpmaif_ctrl *dpmaif_ctrl = rxq->dpmaif_ctrl;
 682         int ret;
 683
 684         queue_work(dpmaif_ctrl->bat_release_wq, &dpmaif_ctrl->bat_release_work);
 685
 686         ret = t7xx_dpmaif_pit_release_and_add(rxq);
 687         if (ret < 0)
 688                 dev_err(dpmaif_ctrl->dev, "RXQ%u update PIT failed: %d\n", rxq->index, ret);
 689
 690         return ret;
 691 }
 692
 693 static void t7xx_dpmaif_rx_skb(struct dpmaif_rx_queue *rxq,
 694                                struct dpmaif_cur_rx_skb_info *skb_info)
 695 {
 696         struct dpmaif_ctrl *dpmaif_ctrl = rxq->dpmaif_ctrl;
 697         struct sk_buff *skb = skb_info->cur_skb;
 698         struct t7xx_skb_cb *skb_cb;
 699         u8 netif_id;
 700
 701         skb_info->cur_skb = NULL;
 702
 703         if (skb_info->pit_dp) {
 704                 dev_kfree_skb_any(skb);
 705                 return;
 706         }
 707
 708         skb->ip_summed = skb_info->check_sum == DPMAIF_CS_RESULT_PASS ? CHECKSUM_UNNECESSARY :
 709                                                                         CHECKSUM_NONE;
 710         netif_id = FIELD_GET(NETIF_MASK, skb_info->cur_chn_idx);
 711         skb_cb = T7XX_SKB_CB(skb);
 712         skb_cb->netif_idx = netif_id;
 713         skb_cb->rx_pkt_type = skb_info->pkt_type;
 714         dpmaif_ctrl->callbacks->recv_skb(dpmaif_ctrl->t7xx_dev->ccmni_ctlb, skb, &rxq->napi);
 715 }
 716
 717 static int t7xx_dpmaif_rx_start(struct dpmaif_rx_queue *rxq, const unsigned int pit_cnt,
 718                                 const unsigned int budget, int *once_more)
 719 {
 720         unsigned int cur_pit, pit_len, rx_cnt, recv_skb_cnt = 0;
 721         struct device *dev = rxq->dpmaif_ctrl->dev;
 722         struct dpmaif_cur_rx_skb_info *skb_info;
 723         int ret = 0;
 724
 725         pit_len = rxq->pit_size_cnt;
 726         skb_info = &rxq->rx_data_info;
 727         cur_pit = rxq->pit_rd_idx;
 728
 729         for (rx_cnt = 0; rx_cnt < pit_cnt; rx_cnt++) {
 730                 struct dpmaif_pit *pkt_info;
 731                 u32 val;
 732
 733                 if (!skb_info->msg_pit_received && recv_skb_cnt >= budget)
 734                         break;
 735
 736                 pkt_info = (struct dpmaif_pit *)rxq->pit_base + cur_pit;
 737                 if (t7xx_dpmaif_check_pit_seq(rxq, pkt_info)) {
 738                         dev_err_ratelimited(dev, "RXQ%u checks PIT SEQ fail\n", rxq->index);
 739                         *once_more = 1;
 740                         return recv_skb_cnt;
 741                 }
 742
 743                 val = FIELD_GET(PD_PIT_PACKET_TYPE, le32_to_cpu(pkt_info->header));
 744                 if (val == DES_PT_MSG) {
 745                         if (skb_info->msg_pit_received)
 746                                 dev_err(dev, "RXQ%u received repeated PIT\n", rxq->index);
 747
 748                         skb_info->msg_pit_received = true;
 749                         t7xx_dpmaif_parse_msg_pit(rxq, pkt_info, skb_info);
 750                 } else { /* DES_PT_PD */
 751                         val = FIELD_GET(PD_PIT_BUFFER_TYPE, le32_to_cpu(pkt_info->header));
 752                         if (val != PKT_BUF_FRAG)
 753                                 ret = t7xx_dpmaif_get_rx_pkt(rxq, pkt_info, skb_info);
 754                         else if (!skb_info->cur_skb)
 755                                 ret = -EINVAL;
 756                         else
 757                                 ret = t7xx_dpmaif_get_frag(rxq, pkt_info, skb_info);
 758
 759                         if (ret < 0) {
 760                                 skb_info->err_payload = 1;
 761                                 dev_err_ratelimited(dev, "RXQ%u error payload\n", rxq->index);
 762                         }
 763
 764                         val = FIELD_GET(PD_PIT_CONT, le32_to_cpu(pkt_info->header));
 765                         if (!val) {
 766                                 if (!skb_info->err_payload) {
 767                                         t7xx_dpmaif_rx_skb(rxq, skb_info);
 768                                 } else if (skb_info->cur_skb) {
 769                                         dev_kfree_skb_any(skb_info->cur_skb);
 770                                         skb_info->cur_skb = NULL;
 771                                 }
 772
 773                                 memset(skb_info, 0, sizeof(*skb_info));
 774                                 recv_skb_cnt++;
 775                         }
 776                 }
 777
 778                 cur_pit = t7xx_ring_buf_get_next_wr_idx(pit_len, cur_pit);
 779                 rxq->pit_rd_idx = cur_pit;
 780                 rxq->pit_remain_release_cnt++;
 781
 782                 if (rx_cnt > 0 && !(rx_cnt % DPMAIF_NOTIFY_RELEASE_COUNT)) {
 783                         ret = t7xx_dpmaifq_rx_notify_hw(rxq);
 784                         if (ret < 0)
 785                                 break;
 786                 }
 787         }
 788
 789         if (!ret)
 790                 ret = t7xx_dpmaifq_rx_notify_hw(rxq);
 791
 792         if (ret)
 793                 return ret;
 794
 795         return recv_skb_cnt;
 796 }
 797
 798 static unsigned int t7xx_dpmaifq_poll_pit(struct dpmaif_rx_queue *rxq)
 799 {
 800         unsigned int hw_wr_idx, pit_cnt;
 801
 802         if (!rxq->que_started)
 803                 return 0;
 804
 805         hw_wr_idx = t7xx_dpmaif_dl_dlq_pit_get_wr_idx(&rxq->dpmaif_ctrl->hw_info, rxq->index);
 806         pit_cnt = t7xx_ring_buf_rd_wr_count(rxq->pit_size_cnt, rxq->pit_rd_idx, hw_wr_idx,
 807                                             DPMAIF_READ);
 808         rxq->pit_wr_idx = hw_wr_idx;
 809         return pit_cnt;
 810 }
 811
 812 static int t7xx_dpmaif_napi_rx_data_collect(struct dpmaif_ctrl *dpmaif_ctrl,
 813                                             const unsigned int q_num,
 814                                             const unsigned int budget, int *once_more)
 815 {
 816         struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[q_num];
 817         unsigned int cnt;
 818         int ret = 0;
 819
 820         cnt = t7xx_dpmaifq_poll_pit(rxq);
 821         if (!cnt)
 822                 return ret;
 823
 824         ret = t7xx_dpmaif_rx_start(rxq, cnt, budget, once_more);
 825         if (ret < 0)
 826                 dev_err(dpmaif_ctrl->dev, "dlq%u rx ERR:%d\n", rxq->index, ret);
 827
 828         return ret;
 829 }
 830
 831 int t7xx_dpmaif_napi_rx_poll(struct napi_struct *napi, const int budget)
 832 {
 833         struct dpmaif_rx_queue *rxq = container_of(napi, struct dpmaif_rx_queue, napi);
 834         struct t7xx_pci_dev *t7xx_dev = rxq->dpmaif_ctrl->t7xx_dev;
 835         int ret, once_more = 0, work_done = 0;
 836
 837         atomic_set(&rxq->rx_processing, 1);
 838         /* Ensure rx_processing is changed to 1 before actually begin RX flow */
 839         smp_mb();
 840
 841         if (!rxq->que_started) {
 842                 atomic_set(&rxq->rx_processing, 0);
 843                 pm_runtime_put_autosuspend(rxq->dpmaif_ctrl->dev);
 844                 dev_err(rxq->dpmaif_ctrl->dev, "Work RXQ: %d has not been started\n", rxq->index);
 845                 return work_done;
 846         }
 847
 848         if (!rxq->sleep_lock_pending)
 849                 t7xx_pci_disable_sleep(t7xx_dev);
 850
 851         ret = try_wait_for_completion(&t7xx_dev->sleep_lock_acquire);
 852         if (!ret) {
 853                 napi_complete_done(napi, work_done);
 854                 rxq->sleep_lock_pending = true;
 855                 napi_schedule(napi);
 856                 return work_done;
 857         }
 858
 859         rxq->sleep_lock_pending = false;
 860         while (work_done < budget) {
 861                 int each_budget = budget - work_done;
 862                 int rx_cnt = t7xx_dpmaif_napi_rx_data_collect(rxq->dpmaif_ctrl, rxq->index,
 863                                                               each_budget, &once_more);
 864                 if (rx_cnt > 0)
 865                         work_done += rx_cnt;
 866                 else
 867                         break;
 868         }
 869
 870         if (once_more) {
 871                 napi_gro_flush(napi, false);
 872                 work_done = budget;
 873                 t7xx_dpmaif_clr_ip_busy_sts(&rxq->dpmaif_ctrl->hw_info);
 874         } else if (work_done < budget) {
 875                 napi_complete_done(napi, work_done);
 876                 t7xx_dpmaif_clr_ip_busy_sts(&rxq->dpmaif_ctrl->hw_info);
 877                 t7xx_dpmaif_dlq_unmask_rx_done(&rxq->dpmaif_ctrl->hw_info, rxq->index);
 878                 t7xx_pci_enable_sleep(rxq->dpmaif_ctrl->t7xx_dev);
 879                 pm_runtime_mark_last_busy(rxq->dpmaif_ctrl->dev);
 880                 pm_runtime_put_autosuspend(rxq->dpmaif_ctrl->dev);
 881                 atomic_set(&rxq->rx_processing, 0);
 882         } else {
 883                 t7xx_dpmaif_clr_ip_busy_sts(&rxq->dpmaif_ctrl->hw_info);
 884         }
 885
 886         return work_done;
 887 }
 888
 889 void t7xx_dpmaif_irq_rx_done(struct dpmaif_ctrl *dpmaif_ctrl, const unsigned int que_mask)
 890 {
 891         struct dpmaif_rx_queue *rxq;
 892         struct dpmaif_ctrl *ctrl;
 893         int qno, ret;
 894
 895         qno = ffs(que_mask) - 1;
 896         if (qno < 0 || qno > DPMAIF_RXQ_NUM - 1) {
 897                 dev_err(dpmaif_ctrl->dev, "Invalid RXQ number: %u\n", qno);
 898                 return;
 899         }
 900
 901         rxq = &dpmaif_ctrl->rxq[qno];
 902         ctrl = rxq->dpmaif_ctrl;
 903         /* We need to make sure that the modem has been resumed before
 904          * calling napi. This can't be done inside the polling function
 905          * as we could be blocked waiting for device to be resumed,
 906          * which can't be done from softirq context the poll function
 907          * is running in.
 908          */
 909         ret = pm_runtime_resume_and_get(ctrl->dev);
 910         if (ret < 0 && ret != -EACCES) {
 911                 dev_err(ctrl->dev, "Failed to resume device: %d\n", ret);
 912                 return;
 913         }
 914         napi_schedule(&rxq->napi);
 915 }
 916
 917 static void t7xx_dpmaif_base_free(const struct dpmaif_ctrl *dpmaif_ctrl,
 918                                   const struct dpmaif_bat_request *bat_req)
 919 {
 920         if (bat_req->bat_base)
 921                 dma_free_coherent(dpmaif_ctrl->dev,
 922                                   bat_req->bat_size_cnt * sizeof(struct dpmaif_bat),
 923                                   bat_req->bat_base, bat_req->bat_bus_addr);
 924 }
 925
 926 /**
 927  * t7xx_dpmaif_bat_alloc() - Allocate the BAT ring buffer.
 928  * @dpmaif_ctrl: Pointer to DPMAIF context structure.
 929  * @bat_req: Pointer to BAT request structure.
 930  * @buf_type: BAT ring type.
 931  *
 932  * This function allocates the BAT ring buffer shared with the HW device, also allocates
 933  * a buffer used to store information about the BAT skbs for further release.
 934  *
 935  * Return:
 936  * * 0          - Success.
 937  * * -ERROR     - Error code.
 938  */
 939 int t7xx_dpmaif_bat_alloc(const struct dpmaif_ctrl *dpmaif_ctrl, struct dpmaif_bat_request *bat_req,
 940                           const enum bat_type buf_type)
 941 {
 942         int sw_buf_size;
 943
 944         if (buf_type == BAT_TYPE_FRAG) {
 945                 sw_buf_size = sizeof(struct dpmaif_bat_page);
 946                 bat_req->bat_size_cnt = DPMAIF_FRG_COUNT;
 947                 bat_req->pkt_buf_sz = DPMAIF_HW_FRG_PKTBUF;
 948         } else {
 949                 sw_buf_size = sizeof(struct dpmaif_bat_skb);
 950                 bat_req->bat_size_cnt = DPMAIF_BAT_COUNT;
 951                 bat_req->pkt_buf_sz = DPMAIF_HW_BAT_PKTBUF;
 952         }
 953
 954         bat_req->type = buf_type;
 955         bat_req->bat_wr_idx = 0;
 956         bat_req->bat_release_rd_idx = 0;
 957
 958         bat_req->bat_base = dma_alloc_coherent(dpmaif_ctrl->dev,
 959                                                bat_req->bat_size_cnt * sizeof(struct dpmaif_bat),
 960                                                &bat_req->bat_bus_addr, GFP_KERNEL | __GFP_ZERO);
 961         if (!bat_req->bat_base)
 962                 return -ENOMEM;
 963
 964         /* For AP SW to record skb information */
 965         bat_req->bat_skb = devm_kzalloc(dpmaif_ctrl->dev, bat_req->bat_size_cnt * sw_buf_size,
 966                                         GFP_KERNEL);
 967         if (!bat_req->bat_skb)
 968                 goto err_free_dma_mem;
 969
 970         bat_req->bat_bitmap = bitmap_zalloc(bat_req->bat_size_cnt, GFP_KERNEL);
 971         if (!bat_req->bat_bitmap)
 972                 goto err_free_dma_mem;
 973
 974         spin_lock_init(&bat_req->mask_lock);
 975         atomic_set(&bat_req->refcnt, 0);
 976         return 0;
 977
 978 err_free_dma_mem:
 979         t7xx_dpmaif_base_free(dpmaif_ctrl, bat_req);
 980
 981         return -ENOMEM;
 982 }
 983
 984 void t7xx_dpmaif_bat_free(const struct dpmaif_ctrl *dpmaif_ctrl, struct dpmaif_bat_request *bat_req)
 985 {
 986         if (!bat_req || !atomic_dec_and_test(&bat_req->refcnt))
 987                 return;
 988
 989         bitmap_free(bat_req->bat_bitmap);
 990         bat_req->bat_bitmap = NULL;
 991
 992         if (bat_req->bat_skb) {
 993                 unsigned int i;
 994
 995                 for (i = 0; i < bat_req->bat_size_cnt; i++) {
 996                         if (bat_req->type == BAT_TYPE_FRAG)
 997                                 t7xx_unmap_bat_page(dpmaif_ctrl->dev, bat_req->bat_skb, i);
 998                         else
 999                                 t7xx_unmap_bat_skb(dpmaif_ctrl->dev, bat_req->bat_skb, i);
1000                 }
1001         }
1002
1003         t7xx_dpmaif_base_free(dpmaif_ctrl, bat_req);
1004 }
1005
1006 static int t7xx_dpmaif_rx_alloc(struct dpmaif_rx_queue *rxq)
1007 {
1008         rxq->pit_size_cnt = DPMAIF_PIT_COUNT;
1009         rxq->pit_rd_idx = 0;
1010         rxq->pit_wr_idx = 0;
1011         rxq->pit_release_rd_idx = 0;
1012         rxq->expect_pit_seq = 0;
1013         rxq->pit_remain_release_cnt = 0;
1014         memset(&rxq->rx_data_info, 0, sizeof(rxq->rx_data_info));
1015
1016         rxq->pit_base = dma_alloc_coherent(rxq->dpmaif_ctrl->dev,
1017                                            rxq->pit_size_cnt * sizeof(struct dpmaif_pit),
1018                                            &rxq->pit_bus_addr, GFP_KERNEL | __GFP_ZERO);
1019         if (!rxq->pit_base)
1020                 return -ENOMEM;
1021
1022         rxq->bat_req = &rxq->dpmaif_ctrl->bat_req;
1023         atomic_inc(&rxq->bat_req->refcnt);
1024
1025         rxq->bat_frag = &rxq->dpmaif_ctrl->bat_frag;
1026         atomic_inc(&rxq->bat_frag->refcnt);
1027         return 0;
1028 }
1029
1030 static void t7xx_dpmaif_rx_buf_free(const struct dpmaif_rx_queue *rxq)
1031 {
1032         if (!rxq->dpmaif_ctrl)
1033                 return;
1034
1035         t7xx_dpmaif_bat_free(rxq->dpmaif_ctrl, rxq->bat_req);
1036         t7xx_dpmaif_bat_free(rxq->dpmaif_ctrl, rxq->bat_frag);
1037
1038         if (rxq->pit_base)
1039                 dma_free_coherent(rxq->dpmaif_ctrl->dev,
1040                                   rxq->pit_size_cnt * sizeof(struct dpmaif_pit),
1041                                   rxq->pit_base, rxq->pit_bus_addr);
1042 }
1043
1044 int t7xx_dpmaif_rxq_init(struct dpmaif_rx_queue *queue)
1045 {
1046         int ret;
1047
1048         ret = t7xx_dpmaif_rx_alloc(queue);
1049         if (ret < 0)
1050                 dev_err(queue->dpmaif_ctrl->dev, "Failed to allocate RX buffers: %d\n", ret);
1051
1052         return ret;
1053 }
1054
1055 void t7xx_dpmaif_rxq_free(struct dpmaif_rx_queue *queue)
1056 {
1057         t7xx_dpmaif_rx_buf_free(queue);
1058 }
1059
1060 static void t7xx_dpmaif_bat_release_work(struct work_struct *work)
1061 {
1062         struct dpmaif_ctrl *dpmaif_ctrl = container_of(work, struct dpmaif_ctrl, bat_release_work);
1063         struct dpmaif_rx_queue *rxq;
1064         int ret;
1065
1066         ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev);
1067         if (ret < 0 && ret != -EACCES)
1068                 return;
1069
1070         t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev);
1071
1072         /* ALL RXQ use one BAT table, so choose DPF_RX_QNO_DFT */
1073         rxq = &dpmaif_ctrl->rxq[DPF_RX_QNO_DFT];
1074         if (t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev)) {
1075                 t7xx_dpmaif_bat_release_and_add(rxq);
1076                 t7xx_dpmaif_frag_bat_release_and_add(rxq);
1077         }
1078
1079         t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev);
1080         pm_runtime_mark_last_busy(dpmaif_ctrl->dev);
1081         pm_runtime_put_autosuspend(dpmaif_ctrl->dev);
1082 }
1083
1084 int t7xx_dpmaif_bat_rel_wq_alloc(struct dpmaif_ctrl *dpmaif_ctrl)
1085 {
1086         dpmaif_ctrl->bat_release_wq = alloc_workqueue("dpmaif_bat_release_work_queue",
1087                                                       WQ_MEM_RECLAIM, 1);
1088         if (!dpmaif_ctrl->bat_release_wq)
1089                 return -ENOMEM;
1090
1091         INIT_WORK(&dpmaif_ctrl->bat_release_work, t7xx_dpmaif_bat_release_work);
1092         return 0;
1093 }
1094
1095 void t7xx_dpmaif_bat_wq_rel(struct dpmaif_ctrl *dpmaif_ctrl)
1096 {
1097         flush_work(&dpmaif_ctrl->bat_release_work);
1098
1099         if (dpmaif_ctrl->bat_release_wq) {
1100                 destroy_workqueue(dpmaif_ctrl->bat_release_wq);
1101                 dpmaif_ctrl->bat_release_wq = NULL;
1102         }
1103 }
1104
1105 /**
1106  * t7xx_dpmaif_rx_stop() - Suspend RX flow.
1107  * @dpmaif_ctrl: Pointer to data path control struct dpmaif_ctrl.
1108  *
1109  * Wait for all the RX work to finish executing and mark the RX queue as paused.
1110  */
1111 void t7xx_dpmaif_rx_stop(struct dpmaif_ctrl *dpmaif_ctrl)
1112 {
1113         unsigned int i;
1114
1115         for (i = 0; i < DPMAIF_RXQ_NUM; i++) {
1116                 struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[i];
1117                 int timeout, value;
1118
1119                 timeout = readx_poll_timeout_atomic(atomic_read, &rxq->rx_processing, value,
1120                                                     !value, 0, DPMAIF_CHECK_INIT_TIMEOUT_US);
1121                 if (timeout)
1122                         dev_err(dpmaif_ctrl->dev, "Stop RX SW failed\n");
1123
1124                 /* Ensure RX processing has stopped before we set rxq->que_started to false */
1125                 smp_mb();
1126                 rxq->que_started = false;
1127         }
1128 }
1129
1130 static void t7xx_dpmaif_stop_rxq(struct dpmaif_rx_queue *rxq)
1131 {
1132         int cnt, j = 0;
1133
1134         rxq->que_started = false;
1135
1136         do {
1137                 cnt = t7xx_ring_buf_rd_wr_count(rxq->pit_size_cnt, rxq->pit_rd_idx,
1138                                                 rxq->pit_wr_idx, DPMAIF_READ);
1139
1140                 if (++j >= DPMAIF_MAX_CHECK_COUNT) {
1141                         dev_err(rxq->dpmaif_ctrl->dev, "Stop RX SW failed, %d\n", cnt);
1142                         break;
1143                 }
1144         } while (cnt);
1145
1146         memset(rxq->pit_base, 0, rxq->pit_size_cnt * sizeof(struct dpmaif_pit));
1147         memset(rxq->bat_req->bat_base, 0, rxq->bat_req->bat_size_cnt * sizeof(struct dpmaif_bat));
1148         bitmap_zero(rxq->bat_req->bat_bitmap, rxq->bat_req->bat_size_cnt);
1149         memset(&rxq->rx_data_info, 0, sizeof(rxq->rx_data_info));
1150
1151         rxq->pit_rd_idx = 0;
1152         rxq->pit_wr_idx = 0;
1153         rxq->pit_release_rd_idx = 0;
1154         rxq->expect_pit_seq = 0;
1155         rxq->pit_remain_release_cnt = 0;
1156         rxq->bat_req->bat_release_rd_idx = 0;
1157         rxq->bat_req->bat_wr_idx = 0;
1158         rxq->bat_frag->bat_release_rd_idx = 0;
1159         rxq->bat_frag->bat_wr_idx = 0;
1160 }
1161
1162 void t7xx_dpmaif_rx_clear(struct dpmaif_ctrl *dpmaif_ctrl)
1163 {
1164         int i;
1165
1166         for (i = 0; i < DPMAIF_RXQ_NUM; i++)
1167                 t7xx_dpmaif_stop_rxq(&dpmaif_ctrl->rxq[i]);
1168 }