drivers/net/wwan/t7xx/t7xx_hif_cldma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2021, MediaTek Inc.
   4  * Copyright (c) 2021-2022, Intel Corporation.
   5  *
   6  * Authors:
   7  *  Amir Hanania <amir.hanania@intel.com>
   8  *  Haijun Liu <haijun.liu@mediatek.com>
   9  *  Moises Veleta <moises.veleta@intel.com>
  10  *  Ricardo Martinez <ricardo.martinez@linux.intel.com>
  11  *  Sreehari Kancharla <sreehari.kancharla@intel.com>
  12  *
  13  * Contributors:
  14  *  Andy Shevchenko <andriy.shevchenko@linux.intel.com>
  15  *  Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com>
  16  *  Eliot Lee <eliot.lee@intel.com>
  17  */
  18
  19 #include <linux/bits.h>
  20 #include <linux/bitops.h>
  21 #include <linux/delay.h>
  22 #include <linux/device.h>
  23 #include <linux/dmapool.h>
  24 #include <linux/dma-mapping.h>
  25 #include <linux/dma-direction.h>
  26 #include <linux/gfp.h>
  27 #include <linux/io.h>
  28 #include <linux/io-64-nonatomic-lo-hi.h>
  29 #include <linux/iopoll.h>
  30 #include <linux/irqreturn.h>
  31 #include <linux/kernel.h>
  32 #include <linux/kthread.h>
  33 #include <linux/list.h>
  34 #include <linux/netdevice.h>
  35 #include <linux/pci.h>
  36 #include <linux/pm_runtime.h>
  37 #include <linux/sched.h>
  38 #include <linux/skbuff.h>
  39 #include <linux/slab.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/types.h>
  42 #include <linux/wait.h>
  43 #include <linux/workqueue.h>
  44
  45 #include "t7xx_cldma.h"
  46 #include "t7xx_hif_cldma.h"
  47 #include "t7xx_mhccif.h"
  48 #include "t7xx_pci.h"
  49 #include "t7xx_pcie_mac.h"
  50 #include "t7xx_port_proxy.h"
  51 #include "t7xx_reg.h"
  52 #include "t7xx_state_monitor.h"
  53
  54 #define MAX_TX_BUDGET                   16
  55 #define MAX_RX_BUDGET                   16
  56
  57 #define CHECK_Q_STOP_TIMEOUT_US         1000000
  58 #define CHECK_Q_STOP_STEP_US            10000
  59
  60 #define CLDMA_JUMBO_BUFF_SZ             (63 * 1024 + sizeof(struct ccci_header))
  61
  62 static void md_cd_queue_struct_reset(struct cldma_queue *queue, struct cldma_ctrl *md_ctrl,
  63                                      enum mtk_txrx tx_rx, unsigned int index)
  64 {
  65         queue->dir = tx_rx;
  66         queue->index = index;
  67         queue->md_ctrl = md_ctrl;
  68         queue->tr_ring = NULL;
  69         queue->tr_done = NULL;
  70         queue->tx_next = NULL;
  71 }
  72
  73 static void md_cd_queue_struct_init(struct cldma_queue *queue, struct cldma_ctrl *md_ctrl,
  74                                     enum mtk_txrx tx_rx, unsigned int index)
  75 {
  76         md_cd_queue_struct_reset(queue, md_ctrl, tx_rx, index);
  77         init_waitqueue_head(&queue->req_wq);
  78         spin_lock_init(&queue->ring_lock);
  79 }
  80
  81 static void t7xx_cldma_gpd_set_data_ptr(struct cldma_gpd *gpd, dma_addr_t data_ptr)
  82 {
  83         gpd->data_buff_bd_ptr_h = cpu_to_le32(upper_32_bits(data_ptr));
  84         gpd->data_buff_bd_ptr_l = cpu_to_le32(lower_32_bits(data_ptr));
  85 }
  86
  87 static void t7xx_cldma_gpd_set_next_ptr(struct cldma_gpd *gpd, dma_addr_t next_ptr)
  88 {
  89         gpd->next_gpd_ptr_h = cpu_to_le32(upper_32_bits(next_ptr));
  90         gpd->next_gpd_ptr_l = cpu_to_le32(lower_32_bits(next_ptr));
  91 }
  92
  93 static int t7xx_cldma_alloc_and_map_skb(struct cldma_ctrl *md_ctrl, struct cldma_request *req,
  94                                         size_t size, gfp_t gfp_mask)
  95 {
  96         req->skb = __dev_alloc_skb(size, gfp_mask);
  97         if (!req->skb)
  98                 return -ENOMEM;
  99
 100         req->mapped_buff = dma_map_single(md_ctrl->dev, req->skb->data, size, DMA_FROM_DEVICE);
 101         if (dma_mapping_error(md_ctrl->dev, req->mapped_buff)) {
 102                 dev_kfree_skb_any(req->skb);
 103                 req->skb = NULL;
 104                 req->mapped_buff = 0;
 105                 dev_err(md_ctrl->dev, "DMA mapping failed\n");
 106                 return -ENOMEM;
 107         }
 108
 109         return 0;
 110 }
 111
 112 static int t7xx_cldma_gpd_rx_from_q(struct cldma_queue *queue, int budget, bool *over_budget)
 113 {
 114         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 115         unsigned int hwo_polling_count = 0;
 116         struct t7xx_cldma_hw *hw_info;
 117         bool rx_not_done = true;
 118         unsigned long flags;
 119         int count = 0;
 120
 121         hw_info = &md_ctrl->hw_info;
 122
 123         do {
 124                 struct cldma_request *req;
 125                 struct cldma_gpd *gpd;
 126                 struct sk_buff *skb;
 127                 int ret;
 128
 129                 req = queue->tr_done;
 130                 if (!req)
 131                         return -ENODATA;
 132
 133                 gpd = req->gpd;
 134                 if ((gpd->flags & GPD_FLAGS_HWO) || !req->skb) {
 135                         dma_addr_t gpd_addr;
 136
 137                         if (!pci_device_is_present(to_pci_dev(md_ctrl->dev))) {
 138                                 dev_err(md_ctrl->dev, "PCIe Link disconnected\n");
 139                                 return -ENODEV;
 140                         }
 141
 142                         gpd_addr = ioread64_lo_hi(hw_info->ap_pdn_base +
 143                                                   REG_CLDMA_DL_CURRENT_ADDRL_0 +
 144                                                   queue->index * sizeof(u64));
 145                         if (req->gpd_addr == gpd_addr || hwo_polling_count++ >= 100)
 146                                 return 0;
 147
 148                         udelay(1);
 149                         continue;
 150                 }
 151
 152                 hwo_polling_count = 0;
 153                 skb = req->skb;
 154
 155                 if (req->mapped_buff) {
 156                         dma_unmap_single(md_ctrl->dev, req->mapped_buff,
 157                                          queue->tr_ring->pkt_size, DMA_FROM_DEVICE);
 158                         req->mapped_buff = 0;
 159                 }
 160
 161                 skb->len = 0;
 162                 skb_reset_tail_pointer(skb);
 163                 skb_put(skb, le16_to_cpu(gpd->data_buff_len));
 164
 165                 ret = md_ctrl->recv_skb(queue, skb);
 166                 /* Break processing, will try again later */
 167                 if (ret < 0)
 168                         return ret;
 169
 170                 req->skb = NULL;
 171                 t7xx_cldma_gpd_set_data_ptr(gpd, 0);
 172
 173                 spin_lock_irqsave(&queue->ring_lock, flags);
 174                 queue->tr_done = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
 175                 spin_unlock_irqrestore(&queue->ring_lock, flags);
 176                 req = queue->rx_refill;
 177
 178                 ret = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, queue->tr_ring->pkt_size, GFP_KERNEL);
 179                 if (ret)
 180                         return ret;
 181
 182                 gpd = req->gpd;
 183                 t7xx_cldma_gpd_set_data_ptr(gpd, req->mapped_buff);
 184                 gpd->data_buff_len = 0;
 185                 gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
 186
 187                 spin_lock_irqsave(&queue->ring_lock, flags);
 188                 queue->rx_refill = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
 189                 spin_unlock_irqrestore(&queue->ring_lock, flags);
 190
 191                 rx_not_done = ++count < budget || !need_resched();
 192         } while (rx_not_done);
 193
 194         *over_budget = true;
 195         return 0;
 196 }
 197
 198 static int t7xx_cldma_gpd_rx_collect(struct cldma_queue *queue, int budget)
 199 {
 200         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 201         struct t7xx_cldma_hw *hw_info;
 202         unsigned int pending_rx_int;
 203         bool over_budget = false;
 204         unsigned long flags;
 205         int ret;
 206
 207         hw_info = &md_ctrl->hw_info;
 208
 209         do {
 210                 ret = t7xx_cldma_gpd_rx_from_q(queue, budget, &over_budget);
 211                 if (ret == -ENODATA)
 212                         return 0;
 213                 else if (ret)
 214                         return ret;
 215
 216                 pending_rx_int = 0;
 217
 218                 spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 219                 if (md_ctrl->rxq_active & BIT(queue->index)) {
 220                         if (!t7xx_cldma_hw_queue_status(hw_info, queue->index, MTK_RX))
 221                                 t7xx_cldma_hw_resume_queue(hw_info, queue->index, MTK_RX);
 222
 223                         pending_rx_int = t7xx_cldma_hw_int_status(hw_info, BIT(queue->index),
 224                                                                   MTK_RX);
 225                         if (pending_rx_int) {
 226                                 t7xx_cldma_hw_rx_done(hw_info, pending_rx_int);
 227
 228                                 if (over_budget) {
 229                                         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 230                                         return -EAGAIN;
 231                                 }
 232                         }
 233                 }
 234                 spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 235         } while (pending_rx_int);
 236
 237         return 0;
 238 }
 239
 240 static void t7xx_cldma_rx_done(struct work_struct *work)
 241 {
 242         struct cldma_queue *queue = container_of(work, struct cldma_queue, cldma_work);
 243         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 244         int value;
 245
 246         value = t7xx_cldma_gpd_rx_collect(queue, queue->budget);
 247         if (value && md_ctrl->rxq_active & BIT(queue->index)) {
 248                 queue_work(queue->worker, &queue->cldma_work);
 249                 return;
 250         }
 251
 252         t7xx_cldma_clear_ip_busy(&md_ctrl->hw_info);
 253         t7xx_cldma_hw_irq_en_txrx(&md_ctrl->hw_info, queue->index, MTK_RX);
 254         t7xx_cldma_hw_irq_en_eq(&md_ctrl->hw_info, queue->index, MTK_RX);
 255         pm_runtime_mark_last_busy(md_ctrl->dev);
 256         pm_runtime_put_autosuspend(md_ctrl->dev);
 257 }
 258
 259 static int t7xx_cldma_gpd_tx_collect(struct cldma_queue *queue)
 260 {
 261         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 262         unsigned int dma_len, count = 0;
 263         struct cldma_request *req;
 264         struct cldma_gpd *gpd;
 265         unsigned long flags;
 266         dma_addr_t dma_free;
 267         struct sk_buff *skb;
 268
 269         while (!kthread_should_stop()) {
 270                 spin_lock_irqsave(&queue->ring_lock, flags);
 271                 req = queue->tr_done;
 272                 if (!req) {
 273                         spin_unlock_irqrestore(&queue->ring_lock, flags);
 274                         break;
 275                 }
 276                 gpd = req->gpd;
 277                 if ((gpd->flags & GPD_FLAGS_HWO) || !req->skb) {
 278                         spin_unlock_irqrestore(&queue->ring_lock, flags);
 279                         break;
 280                 }
 281                 queue->budget++;
 282                 dma_free = req->mapped_buff;
 283                 dma_len = le16_to_cpu(gpd->data_buff_len);
 284                 skb = req->skb;
 285                 req->skb = NULL;
 286                 queue->tr_done = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
 287                 spin_unlock_irqrestore(&queue->ring_lock, flags);
 288
 289                 count++;
 290                 dma_unmap_single(md_ctrl->dev, dma_free, dma_len, DMA_TO_DEVICE);
 291                 dev_kfree_skb_any(skb);
 292         }
 293
 294         if (count)
 295                 wake_up_nr(&queue->req_wq, count);
 296
 297         return count;
 298 }
 299
 300 static void t7xx_cldma_txq_empty_hndl(struct cldma_queue *queue)
 301 {
 302         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 303         struct cldma_request *req;
 304         dma_addr_t ul_curr_addr;
 305         unsigned long flags;
 306         bool pending_gpd;
 307
 308         if (!(md_ctrl->txq_active & BIT(queue->index)))
 309                 return;
 310
 311         spin_lock_irqsave(&queue->ring_lock, flags);
 312         req = list_prev_entry_circular(queue->tx_next, &queue->tr_ring->gpd_ring, entry);
 313         spin_unlock_irqrestore(&queue->ring_lock, flags);
 314
 315         pending_gpd = (req->gpd->flags & GPD_FLAGS_HWO) && req->skb;
 316
 317         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 318         if (pending_gpd) {
 319                 struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 320
 321                 /* Check current processing TGPD, 64-bit address is in a table by Q index */
 322                 ul_curr_addr = ioread64_lo_hi(hw_info->ap_pdn_base + REG_CLDMA_UL_CURRENT_ADDRL_0 +
 323                                               queue->index * sizeof(u64));
 324                 if (req->gpd_addr != ul_curr_addr) {
 325                         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 326                         dev_err(md_ctrl->dev, "CLDMA%d queue %d is not empty\n",
 327                                 md_ctrl->hif_id, queue->index);
 328                         return;
 329                 }
 330
 331                 t7xx_cldma_hw_resume_queue(hw_info, queue->index, MTK_TX);
 332         }
 333         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 334 }
 335
 336 static void t7xx_cldma_tx_done(struct work_struct *work)
 337 {
 338         struct cldma_queue *queue = container_of(work, struct cldma_queue, cldma_work);
 339         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 340         struct t7xx_cldma_hw *hw_info;
 341         unsigned int l2_tx_int;
 342         unsigned long flags;
 343
 344         hw_info = &md_ctrl->hw_info;
 345         t7xx_cldma_gpd_tx_collect(queue);
 346         l2_tx_int = t7xx_cldma_hw_int_status(hw_info, BIT(queue->index) | EQ_STA_BIT(queue->index),
 347                                              MTK_TX);
 348         if (l2_tx_int & EQ_STA_BIT(queue->index)) {
 349                 t7xx_cldma_hw_tx_done(hw_info, EQ_STA_BIT(queue->index));
 350                 t7xx_cldma_txq_empty_hndl(queue);
 351         }
 352
 353         if (l2_tx_int & BIT(queue->index)) {
 354                 t7xx_cldma_hw_tx_done(hw_info, BIT(queue->index));
 355                 queue_work(queue->worker, &queue->cldma_work);
 356                 return;
 357         }
 358
 359         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 360         if (md_ctrl->txq_active & BIT(queue->index)) {
 361                 t7xx_cldma_clear_ip_busy(hw_info);
 362                 t7xx_cldma_hw_irq_en_eq(hw_info, queue->index, MTK_TX);
 363                 t7xx_cldma_hw_irq_en_txrx(hw_info, queue->index, MTK_TX);
 364         }
 365         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 366
 367         pm_runtime_mark_last_busy(md_ctrl->dev);
 368         pm_runtime_put_autosuspend(md_ctrl->dev);
 369 }
 370
 371 static void t7xx_cldma_ring_free(struct cldma_ctrl *md_ctrl,
 372                                  struct cldma_ring *ring, enum dma_data_direction tx_rx)
 373 {
 374         struct cldma_request *req_cur, *req_next;
 375
 376         list_for_each_entry_safe(req_cur, req_next, &ring->gpd_ring, entry) {
 377                 if (req_cur->mapped_buff && req_cur->skb) {
 378                         dma_unmap_single(md_ctrl->dev, req_cur->mapped_buff,
 379                                          ring->pkt_size, tx_rx);
 380                         req_cur->mapped_buff = 0;
 381                 }
 382
 383                 dev_kfree_skb_any(req_cur->skb);
 384
 385                 if (req_cur->gpd)
 386                         dma_pool_free(md_ctrl->gpd_dmapool, req_cur->gpd, req_cur->gpd_addr);
 387
 388                 list_del(&req_cur->entry);
 389                 kfree(req_cur);
 390         }
 391 }
 392
 393 static struct cldma_request *t7xx_alloc_rx_request(struct cldma_ctrl *md_ctrl, size_t pkt_size)
 394 {
 395         struct cldma_request *req;
 396         int val;
 397
 398         req = kzalloc(sizeof(*req), GFP_KERNEL);
 399         if (!req)
 400                 return NULL;
 401
 402         req->gpd = dma_pool_zalloc(md_ctrl->gpd_dmapool, GFP_KERNEL, &req->gpd_addr);
 403         if (!req->gpd)
 404                 goto err_free_req;
 405
 406         val = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, pkt_size, GFP_KERNEL);
 407         if (val)
 408                 goto err_free_pool;
 409
 410         return req;
 411
 412 err_free_pool:
 413         dma_pool_free(md_ctrl->gpd_dmapool, req->gpd, req->gpd_addr);
 414
 415 err_free_req:
 416         kfree(req);
 417
 418         return NULL;
 419 }
 420
 421 static int t7xx_cldma_rx_ring_init(struct cldma_ctrl *md_ctrl, struct cldma_ring *ring)
 422 {
 423         struct cldma_request *req;
 424         struct cldma_gpd *gpd;
 425         int i;
 426
 427         INIT_LIST_HEAD(&ring->gpd_ring);
 428         ring->length = MAX_RX_BUDGET;
 429
 430         for (i = 0; i < ring->length; i++) {
 431                 req = t7xx_alloc_rx_request(md_ctrl, ring->pkt_size);
 432                 if (!req) {
 433                         t7xx_cldma_ring_free(md_ctrl, ring, DMA_FROM_DEVICE);
 434                         return -ENOMEM;
 435                 }
 436
 437                 gpd = req->gpd;
 438                 t7xx_cldma_gpd_set_data_ptr(gpd, req->mapped_buff);
 439                 gpd->rx_data_allow_len = cpu_to_le16(ring->pkt_size);
 440                 gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
 441                 INIT_LIST_HEAD(&req->entry);
 442                 list_add_tail(&req->entry, &ring->gpd_ring);
 443         }
 444
 445         /* Link previous GPD to next GPD, circular */
 446         list_for_each_entry(req, &ring->gpd_ring, entry) {
 447                 t7xx_cldma_gpd_set_next_ptr(gpd, req->gpd_addr);
 448                 gpd = req->gpd;
 449         }
 450
 451         return 0;
 452 }
 453
 454 static struct cldma_request *t7xx_alloc_tx_request(struct cldma_ctrl *md_ctrl)
 455 {
 456         struct cldma_request *req;
 457
 458         req = kzalloc(sizeof(*req), GFP_KERNEL);
 459         if (!req)
 460                 return NULL;
 461
 462         req->gpd = dma_pool_zalloc(md_ctrl->gpd_dmapool, GFP_KERNEL, &req->gpd_addr);
 463         if (!req->gpd) {
 464                 kfree(req);
 465                 return NULL;
 466         }
 467
 468         return req;
 469 }
 470
 471 static int t7xx_cldma_tx_ring_init(struct cldma_ctrl *md_ctrl, struct cldma_ring *ring)
 472 {
 473         struct cldma_request *req;
 474         struct cldma_gpd *gpd;
 475         int i;
 476
 477         INIT_LIST_HEAD(&ring->gpd_ring);
 478         ring->length = MAX_TX_BUDGET;
 479
 480         for (i = 0; i < ring->length; i++) {
 481                 req = t7xx_alloc_tx_request(md_ctrl);
 482                 if (!req) {
 483                         t7xx_cldma_ring_free(md_ctrl, ring, DMA_TO_DEVICE);
 484                         return -ENOMEM;
 485                 }
 486
 487                 gpd = req->gpd;
 488                 gpd->flags = GPD_FLAGS_IOC;
 489                 INIT_LIST_HEAD(&req->entry);
 490                 list_add_tail(&req->entry, &ring->gpd_ring);
 491         }
 492
 493         /* Link previous GPD to next GPD, circular */
 494         list_for_each_entry(req, &ring->gpd_ring, entry) {
 495                 t7xx_cldma_gpd_set_next_ptr(gpd, req->gpd_addr);
 496                 gpd = req->gpd;
 497         }
 498
 499         return 0;
 500 }
 501
 502 /**
 503  * t7xx_cldma_q_reset() - Reset CLDMA request pointers to their initial values.
 504  * @queue: Pointer to the queue structure.
 505  *
 506  * Called with ring_lock (unless called during initialization phase)
 507  */
 508 static void t7xx_cldma_q_reset(struct cldma_queue *queue)
 509 {
 510         struct cldma_request *req;
 511
 512         req = list_first_entry(&queue->tr_ring->gpd_ring, struct cldma_request, entry);
 513         queue->tr_done = req;
 514         queue->budget = queue->tr_ring->length;
 515
 516         if (queue->dir == MTK_TX)
 517                 queue->tx_next = req;
 518         else
 519                 queue->rx_refill = req;
 520 }
 521
 522 static void t7xx_cldma_rxq_init(struct cldma_queue *queue)
 523 {
 524         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 525
 526         queue->dir = MTK_RX;
 527         queue->tr_ring = &md_ctrl->rx_ring[queue->index];
 528         t7xx_cldma_q_reset(queue);
 529 }
 530
 531 static void t7xx_cldma_txq_init(struct cldma_queue *queue)
 532 {
 533         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 534
 535         queue->dir = MTK_TX;
 536         queue->tr_ring = &md_ctrl->tx_ring[queue->index];
 537         t7xx_cldma_q_reset(queue);
 538 }
 539
 540 static void t7xx_cldma_enable_irq(struct cldma_ctrl *md_ctrl)
 541 {
 542         t7xx_pcie_mac_set_int(md_ctrl->t7xx_dev, md_ctrl->hw_info.phy_interrupt_id);
 543 }
 544
 545 static void t7xx_cldma_disable_irq(struct cldma_ctrl *md_ctrl)
 546 {
 547         t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, md_ctrl->hw_info.phy_interrupt_id);
 548 }
 549
 550 static void t7xx_cldma_irq_work_cb(struct cldma_ctrl *md_ctrl)
 551 {
 552         unsigned long l2_tx_int_msk, l2_rx_int_msk, l2_tx_int, l2_rx_int, val;
 553         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 554         int i;
 555
 556         /* L2 raw interrupt status */
 557         l2_tx_int = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2TISAR0);
 558         l2_rx_int = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2RISAR0);
 559         l2_tx_int_msk = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2TIMR0);
 560         l2_rx_int_msk = ioread32(hw_info->ap_ao_base + REG_CLDMA_L2RIMR0);
 561         l2_tx_int &= ~l2_tx_int_msk;
 562         l2_rx_int &= ~l2_rx_int_msk;
 563
 564         if (l2_tx_int) {
 565                 if (l2_tx_int & (TQ_ERR_INT_BITMASK | TQ_ACTIVE_START_ERR_INT_BITMASK)) {
 566                         /* Read and clear L3 TX interrupt status */
 567                         val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3TISAR0);
 568                         iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3TISAR0);
 569                         val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3TISAR1);
 570                         iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3TISAR1);
 571                 }
 572
 573                 t7xx_cldma_hw_tx_done(hw_info, l2_tx_int);
 574                 if (l2_tx_int & (TXRX_STATUS_BITMASK | EMPTY_STATUS_BITMASK)) {
 575                         for_each_set_bit(i, &l2_tx_int, L2_INT_BIT_COUNT) {
 576                                 if (i < CLDMA_TXQ_NUM) {
 577                                         pm_runtime_get(md_ctrl->dev);
 578                                         t7xx_cldma_hw_irq_dis_eq(hw_info, i, MTK_TX);
 579                                         t7xx_cldma_hw_irq_dis_txrx(hw_info, i, MTK_TX);
 580                                         queue_work(md_ctrl->txq[i].worker,
 581                                                    &md_ctrl->txq[i].cldma_work);
 582                                 } else {
 583                                         t7xx_cldma_txq_empty_hndl(&md_ctrl->txq[i - CLDMA_TXQ_NUM]);
 584                                 }
 585                         }
 586                 }
 587         }
 588
 589         if (l2_rx_int) {
 590                 if (l2_rx_int & (RQ_ERR_INT_BITMASK | RQ_ACTIVE_START_ERR_INT_BITMASK)) {
 591                         /* Read and clear L3 RX interrupt status */
 592                         val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3RISAR0);
 593                         iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3RISAR0);
 594                         val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3RISAR1);
 595                         iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3RISAR1);
 596                 }
 597
 598                 t7xx_cldma_hw_rx_done(hw_info, l2_rx_int);
 599                 if (l2_rx_int & (TXRX_STATUS_BITMASK | EMPTY_STATUS_BITMASK)) {
 600                         l2_rx_int |= l2_rx_int >> CLDMA_RXQ_NUM;
 601                         for_each_set_bit(i, &l2_rx_int, CLDMA_RXQ_NUM) {
 602                                 pm_runtime_get(md_ctrl->dev);
 603                                 t7xx_cldma_hw_irq_dis_eq(hw_info, i, MTK_RX);
 604                                 t7xx_cldma_hw_irq_dis_txrx(hw_info, i, MTK_RX);
 605                                 queue_work(md_ctrl->rxq[i].worker, &md_ctrl->rxq[i].cldma_work);
 606                         }
 607                 }
 608         }
 609 }
 610
 611 static bool t7xx_cldma_qs_are_active(struct cldma_ctrl *md_ctrl)
 612 {
 613         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 614         unsigned int tx_active;
 615         unsigned int rx_active;
 616
 617         if (!pci_device_is_present(to_pci_dev(md_ctrl->dev)))
 618                 return false;
 619
 620         tx_active = t7xx_cldma_hw_queue_status(hw_info, CLDMA_ALL_Q, MTK_TX);
 621         rx_active = t7xx_cldma_hw_queue_status(hw_info, CLDMA_ALL_Q, MTK_RX);
 622
 623         return tx_active || rx_active;
 624 }
 625
 626 /**
 627  * t7xx_cldma_stop() - Stop CLDMA.
 628  * @md_ctrl: CLDMA context structure.
 629  *
 630  * Stop TX and RX queues. Disable L1 and L2 interrupts.
 631  * Clear status registers.
 632  *
 633  * Return:
 634  * * 0          - Success.
 635  * * -ERROR     - Error code from polling cldma_queues_active.
 636  */
 637 int t7xx_cldma_stop(struct cldma_ctrl *md_ctrl)
 638 {
 639         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 640         bool active;
 641         int i, ret;
 642
 643         md_ctrl->rxq_active = 0;
 644         t7xx_cldma_hw_stop_all_qs(hw_info, MTK_RX);
 645         md_ctrl->txq_active = 0;
 646         t7xx_cldma_hw_stop_all_qs(hw_info, MTK_TX);
 647         md_ctrl->txq_started = 0;
 648         t7xx_cldma_disable_irq(md_ctrl);
 649         t7xx_cldma_hw_stop(hw_info, MTK_RX);
 650         t7xx_cldma_hw_stop(hw_info, MTK_TX);
 651         t7xx_cldma_hw_tx_done(hw_info, CLDMA_L2TISAR0_ALL_INT_MASK);
 652         t7xx_cldma_hw_rx_done(hw_info, CLDMA_L2RISAR0_ALL_INT_MASK);
 653
 654         if (md_ctrl->is_late_init) {
 655                 for (i = 0; i < CLDMA_TXQ_NUM; i++)
 656                         flush_work(&md_ctrl->txq[i].cldma_work);
 657
 658                 for (i = 0; i < CLDMA_RXQ_NUM; i++)
 659                         flush_work(&md_ctrl->rxq[i].cldma_work);
 660         }
 661
 662         ret = read_poll_timeout(t7xx_cldma_qs_are_active, active, !active, CHECK_Q_STOP_STEP_US,
 663                                 CHECK_Q_STOP_TIMEOUT_US, true, md_ctrl);
 664         if (ret)
 665                 dev_err(md_ctrl->dev, "Could not stop CLDMA%d queues", md_ctrl->hif_id);
 666
 667         return ret;
 668 }
 669
 670 static void t7xx_cldma_late_release(struct cldma_ctrl *md_ctrl)
 671 {
 672         int i;
 673
 674         if (!md_ctrl->is_late_init)
 675                 return;
 676
 677         for (i = 0; i < CLDMA_TXQ_NUM; i++)
 678                 t7xx_cldma_ring_free(md_ctrl, &md_ctrl->tx_ring[i], DMA_TO_DEVICE);
 679
 680         for (i = 0; i < CLDMA_RXQ_NUM; i++)
 681                 t7xx_cldma_ring_free(md_ctrl, &md_ctrl->rx_ring[i], DMA_FROM_DEVICE);
 682
 683         dma_pool_destroy(md_ctrl->gpd_dmapool);
 684         md_ctrl->gpd_dmapool = NULL;
 685         md_ctrl->is_late_init = false;
 686 }
 687
 688 void t7xx_cldma_reset(struct cldma_ctrl *md_ctrl)
 689 {
 690         unsigned long flags;
 691         int i;
 692
 693         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 694         md_ctrl->txq_active = 0;
 695         md_ctrl->rxq_active = 0;
 696         t7xx_cldma_disable_irq(md_ctrl);
 697         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 698
 699         for (i = 0; i < CLDMA_TXQ_NUM; i++) {
 700                 cancel_work_sync(&md_ctrl->txq[i].cldma_work);
 701
 702                 spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 703                 md_cd_queue_struct_reset(&md_ctrl->txq[i], md_ctrl, MTK_TX, i);
 704                 spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 705         }
 706
 707         for (i = 0; i < CLDMA_RXQ_NUM; i++) {
 708                 cancel_work_sync(&md_ctrl->rxq[i].cldma_work);
 709
 710                 spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 711                 md_cd_queue_struct_reset(&md_ctrl->rxq[i], md_ctrl, MTK_RX, i);
 712                 spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 713         }
 714
 715         t7xx_cldma_late_release(md_ctrl);
 716 }
 717
 718 /**
 719  * t7xx_cldma_start() - Start CLDMA.
 720  * @md_ctrl: CLDMA context structure.
 721  *
 722  * Set TX/RX start address.
 723  * Start all RX queues and enable L2 interrupt.
 724  */
 725 void t7xx_cldma_start(struct cldma_ctrl *md_ctrl)
 726 {
 727         unsigned long flags;
 728
 729         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 730         if (md_ctrl->is_late_init) {
 731                 struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 732                 int i;
 733
 734                 t7xx_cldma_enable_irq(md_ctrl);
 735
 736                 for (i = 0; i < CLDMA_TXQ_NUM; i++) {
 737                         if (md_ctrl->txq[i].tr_done)
 738                                 t7xx_cldma_hw_set_start_addr(hw_info, i,
 739                                                              md_ctrl->txq[i].tr_done->gpd_addr,
 740                                                              MTK_TX);
 741                 }
 742
 743                 for (i = 0; i < CLDMA_RXQ_NUM; i++) {
 744                         if (md_ctrl->rxq[i].tr_done)
 745                                 t7xx_cldma_hw_set_start_addr(hw_info, i,
 746                                                              md_ctrl->rxq[i].tr_done->gpd_addr,
 747                                                              MTK_RX);
 748                 }
 749
 750                 /* Enable L2 interrupt */
 751                 t7xx_cldma_hw_start_queue(hw_info, CLDMA_ALL_Q, MTK_RX);
 752                 t7xx_cldma_hw_start(hw_info);
 753                 md_ctrl->txq_started = 0;
 754                 md_ctrl->txq_active |= TXRX_STATUS_BITMASK;
 755                 md_ctrl->rxq_active |= TXRX_STATUS_BITMASK;
 756         }
 757         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 758 }
 759
 760 static void t7xx_cldma_clear_txq(struct cldma_ctrl *md_ctrl, int qnum)
 761 {
 762         struct cldma_queue *txq = &md_ctrl->txq[qnum];
 763         struct cldma_request *req;
 764         struct cldma_gpd *gpd;
 765         unsigned long flags;
 766
 767         spin_lock_irqsave(&txq->ring_lock, flags);
 768         t7xx_cldma_q_reset(txq);
 769         list_for_each_entry(req, &txq->tr_ring->gpd_ring, entry) {
 770                 gpd = req->gpd;
 771                 gpd->flags &= ~GPD_FLAGS_HWO;
 772                 t7xx_cldma_gpd_set_data_ptr(gpd, 0);
 773                 gpd->data_buff_len = 0;
 774                 dev_kfree_skb_any(req->skb);
 775                 req->skb = NULL;
 776         }
 777         spin_unlock_irqrestore(&txq->ring_lock, flags);
 778 }
 779
 780 static int t7xx_cldma_clear_rxq(struct cldma_ctrl *md_ctrl, int qnum)
 781 {
 782         struct cldma_queue *rxq = &md_ctrl->rxq[qnum];
 783         struct cldma_request *req;
 784         struct cldma_gpd *gpd;
 785         unsigned long flags;
 786         int ret = 0;
 787
 788         spin_lock_irqsave(&rxq->ring_lock, flags);
 789         t7xx_cldma_q_reset(rxq);
 790         list_for_each_entry(req, &rxq->tr_ring->gpd_ring, entry) {
 791                 gpd = req->gpd;
 792                 gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
 793                 gpd->data_buff_len = 0;
 794
 795                 if (req->skb) {
 796                         req->skb->len = 0;
 797                         skb_reset_tail_pointer(req->skb);
 798                 }
 799         }
 800
 801         list_for_each_entry(req, &rxq->tr_ring->gpd_ring, entry) {
 802                 if (req->skb)
 803                         continue;
 804
 805                 ret = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, rxq->tr_ring->pkt_size, GFP_ATOMIC);
 806                 if (ret)
 807                         break;
 808
 809                 t7xx_cldma_gpd_set_data_ptr(req->gpd, req->mapped_buff);
 810         }
 811         spin_unlock_irqrestore(&rxq->ring_lock, flags);
 812
 813         return ret;
 814 }
 815
 816 void t7xx_cldma_clear_all_qs(struct cldma_ctrl *md_ctrl, enum mtk_txrx tx_rx)
 817 {
 818         int i;
 819
 820         if (tx_rx == MTK_TX) {
 821                 for (i = 0; i < CLDMA_TXQ_NUM; i++)
 822                         t7xx_cldma_clear_txq(md_ctrl, i);
 823         } else {
 824                 for (i = 0; i < CLDMA_RXQ_NUM; i++)
 825                         t7xx_cldma_clear_rxq(md_ctrl, i);
 826         }
 827 }
 828
 829 void t7xx_cldma_stop_all_qs(struct cldma_ctrl *md_ctrl, enum mtk_txrx tx_rx)
 830 {
 831         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 832         unsigned long flags;
 833
 834         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 835         t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, tx_rx);
 836         t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, tx_rx);
 837         if (tx_rx == MTK_RX)
 838                 md_ctrl->rxq_active &= ~TXRX_STATUS_BITMASK;
 839         else
 840                 md_ctrl->txq_active &= ~TXRX_STATUS_BITMASK;
 841         t7xx_cldma_hw_stop_all_qs(hw_info, tx_rx);
 842         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 843 }
 844
 845 static int t7xx_cldma_gpd_handle_tx_request(struct cldma_queue *queue, struct cldma_request *tx_req,
 846                                             struct sk_buff *skb)
 847 {
 848         struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 849         struct cldma_gpd *gpd = tx_req->gpd;
 850         unsigned long flags;
 851
 852         /* Update GPD */
 853         tx_req->mapped_buff = dma_map_single(md_ctrl->dev, skb->data, skb->len, DMA_TO_DEVICE);
 854
 855         if (dma_mapping_error(md_ctrl->dev, tx_req->mapped_buff)) {
 856                 dev_err(md_ctrl->dev, "DMA mapping failed\n");
 857                 return -ENOMEM;
 858         }
 859
 860         t7xx_cldma_gpd_set_data_ptr(gpd, tx_req->mapped_buff);
 861         gpd->data_buff_len = cpu_to_le16(skb->len);
 862
 863         /* This lock must cover TGPD setting, as even without a resume operation,
 864          * CLDMA can send next HWO=1 if last TGPD just finished.
 865          */
 866         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 867         if (md_ctrl->txq_active & BIT(queue->index))
 868                 gpd->flags |= GPD_FLAGS_HWO;
 869
 870         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 871
 872         tx_req->skb = skb;
 873         return 0;
 874 }
 875
 876 /* Called with cldma_lock */
 877 static void t7xx_cldma_hw_start_send(struct cldma_ctrl *md_ctrl, int qno,
 878                                      struct cldma_request *prev_req)
 879 {
 880         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 881
 882         /* Check whether the device was powered off (CLDMA start address is not set) */
 883         if (!t7xx_cldma_tx_addr_is_set(hw_info, qno)) {
 884                 t7xx_cldma_hw_init(hw_info);
 885                 t7xx_cldma_hw_set_start_addr(hw_info, qno, prev_req->gpd_addr, MTK_TX);
 886                 md_ctrl->txq_started &= ~BIT(qno);
 887         }
 888
 889         if (!t7xx_cldma_hw_queue_status(hw_info, qno, MTK_TX)) {
 890                 if (md_ctrl->txq_started & BIT(qno))
 891                         t7xx_cldma_hw_resume_queue(hw_info, qno, MTK_TX);
 892                 else
 893                         t7xx_cldma_hw_start_queue(hw_info, qno, MTK_TX);
 894
 895                 md_ctrl->txq_started |= BIT(qno);
 896         }
 897 }
 898
 899 /**
 900  * t7xx_cldma_set_recv_skb() - Set the callback to handle RX packets.
 901  * @md_ctrl: CLDMA context structure.
 902  * @recv_skb: Receiving skb callback.
 903  */
 904 void t7xx_cldma_set_recv_skb(struct cldma_ctrl *md_ctrl,
 905                              int (*recv_skb)(struct cldma_queue *queue, struct sk_buff *skb))
 906 {
 907         md_ctrl->recv_skb = recv_skb;
 908 }
 909
 910 /**
 911  * t7xx_cldma_send_skb() - Send control data to modem.
 912  * @md_ctrl: CLDMA context structure.
 913  * @qno: Queue number.
 914  * @skb: Socket buffer.
 915  *
 916  * Return:
 917  * * 0          - Success.
 918  * * -ENOMEM    - Allocation failure.
 919  * * -EINVAL    - Invalid queue request.
 920  * * -EIO       - Queue is not active.
 921  * * -ETIMEDOUT - Timeout waiting for the device to wake up.
 922  */
 923 int t7xx_cldma_send_skb(struct cldma_ctrl *md_ctrl, int qno, struct sk_buff *skb)
 924 {
 925         struct cldma_request *tx_req;
 926         struct cldma_queue *queue;
 927         unsigned long flags;
 928         int ret;
 929
 930         if (qno >= CLDMA_TXQ_NUM)
 931                 return -EINVAL;
 932
 933         ret = pm_runtime_resume_and_get(md_ctrl->dev);
 934         if (ret < 0 && ret != -EACCES)
 935                 return ret;
 936
 937         t7xx_pci_disable_sleep(md_ctrl->t7xx_dev);
 938         queue = &md_ctrl->txq[qno];
 939
 940         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 941         if (!(md_ctrl->txq_active & BIT(qno))) {
 942                 ret = -EIO;
 943                 spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 944                 goto allow_sleep;
 945         }
 946         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 947
 948         do {
 949                 spin_lock_irqsave(&queue->ring_lock, flags);
 950                 tx_req = queue->tx_next;
 951                 if (queue->budget > 0 && !tx_req->skb) {
 952                         struct list_head *gpd_ring = &queue->tr_ring->gpd_ring;
 953
 954                         queue->budget--;
 955                         t7xx_cldma_gpd_handle_tx_request(queue, tx_req, skb);
 956                         queue->tx_next = list_next_entry_circular(tx_req, gpd_ring, entry);
 957                         spin_unlock_irqrestore(&queue->ring_lock, flags);
 958
 959                         if (!t7xx_pci_sleep_disable_complete(md_ctrl->t7xx_dev)) {
 960                                 ret = -ETIMEDOUT;
 961                                 break;
 962                         }
 963
 964                         /* Protect the access to the modem for queues operations (resume/start)
 965                          * which access shared locations by all the queues.
 966                          * cldma_lock is independent of ring_lock which is per queue.
 967                          */
 968                         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 969                         t7xx_cldma_hw_start_send(md_ctrl, qno, tx_req);
 970                         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 971
 972                         break;
 973                 }
 974                 spin_unlock_irqrestore(&queue->ring_lock, flags);
 975
 976                 if (!t7xx_pci_sleep_disable_complete(md_ctrl->t7xx_dev)) {
 977                         ret = -ETIMEDOUT;
 978                         break;
 979                 }
 980
 981                 if (!t7xx_cldma_hw_queue_status(&md_ctrl->hw_info, qno, MTK_TX)) {
 982                         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 983                         t7xx_cldma_hw_resume_queue(&md_ctrl->hw_info, qno, MTK_TX);
 984                         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 985                 }
 986
 987                 ret = wait_event_interruptible_exclusive(queue->req_wq, queue->budget > 0);
 988         } while (!ret);
 989
 990 allow_sleep:
 991         t7xx_pci_enable_sleep(md_ctrl->t7xx_dev);
 992         pm_runtime_mark_last_busy(md_ctrl->dev);
 993         pm_runtime_put_autosuspend(md_ctrl->dev);
 994         return ret;
 995 }
 996
 997 static int t7xx_cldma_late_init(struct cldma_ctrl *md_ctrl)
 998 {
 999         char dma_pool_name[32];
1000         int i, j, ret;
1001
1002         if (md_ctrl->is_late_init) {
1003                 dev_err(md_ctrl->dev, "CLDMA late init was already done\n");
1004                 return -EALREADY;
1005         }
1006
1007         snprintf(dma_pool_name, sizeof(dma_pool_name), "cldma_req_hif%d", md_ctrl->hif_id);
1008
1009         md_ctrl->gpd_dmapool = dma_pool_create(dma_pool_name, md_ctrl->dev,
1010                                                sizeof(struct cldma_gpd), GPD_DMAPOOL_ALIGN, 0);
1011         if (!md_ctrl->gpd_dmapool) {
1012                 dev_err(md_ctrl->dev, "DMA pool alloc fail\n");
1013                 return -ENOMEM;
1014         }
1015
1016         for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1017                 ret = t7xx_cldma_tx_ring_init(md_ctrl, &md_ctrl->tx_ring[i]);
1018                 if (ret) {
1019                         dev_err(md_ctrl->dev, "control TX ring init fail\n");
1020                         goto err_free_tx_ring;
1021                 }
1022
1023                 md_ctrl->tx_ring[i].pkt_size = CLDMA_MTU;
1024         }
1025
1026         for (j = 0; j < CLDMA_RXQ_NUM; j++) {
1027                 md_ctrl->rx_ring[j].pkt_size = CLDMA_MTU;
1028
1029                 if (j == CLDMA_RXQ_NUM - 1)
1030                         md_ctrl->rx_ring[j].pkt_size = CLDMA_JUMBO_BUFF_SZ;
1031
1032                 ret = t7xx_cldma_rx_ring_init(md_ctrl, &md_ctrl->rx_ring[j]);
1033                 if (ret) {
1034                         dev_err(md_ctrl->dev, "Control RX ring init fail\n");
1035                         goto err_free_rx_ring;
1036                 }
1037         }
1038
1039         for (i = 0; i < CLDMA_TXQ_NUM; i++)
1040                 t7xx_cldma_txq_init(&md_ctrl->txq[i]);
1041
1042         for (j = 0; j < CLDMA_RXQ_NUM; j++)
1043                 t7xx_cldma_rxq_init(&md_ctrl->rxq[j]);
1044
1045         md_ctrl->is_late_init = true;
1046         return 0;
1047
1048 err_free_rx_ring:
1049         while (j--)
1050                 t7xx_cldma_ring_free(md_ctrl, &md_ctrl->rx_ring[j], DMA_FROM_DEVICE);
1051
1052 err_free_tx_ring:
1053         while (i--)
1054                 t7xx_cldma_ring_free(md_ctrl, &md_ctrl->tx_ring[i], DMA_TO_DEVICE);
1055
1056         return ret;
1057 }
1058
1059 static void __iomem *t7xx_pcie_addr_transfer(void __iomem *addr, u32 addr_trs1, u32 phy_addr)
1060 {
1061         return addr + phy_addr - addr_trs1;
1062 }
1063
1064 static void t7xx_hw_info_init(struct cldma_ctrl *md_ctrl)
1065 {
1066         struct t7xx_addr_base *pbase = &md_ctrl->t7xx_dev->base_addr;
1067         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1068         u32 phy_ao_base, phy_pd_base;
1069
1070         hw_info->hw_mode = MODE_BIT_64;
1071
1072         if (md_ctrl->hif_id == CLDMA_ID_MD) {
1073                 phy_ao_base = CLDMA1_AO_BASE;
1074                 phy_pd_base = CLDMA1_PD_BASE;
1075                 hw_info->phy_interrupt_id = CLDMA1_INT;
1076         } else {
1077                 phy_ao_base = CLDMA0_AO_BASE;
1078                 phy_pd_base = CLDMA0_PD_BASE;
1079                 hw_info->phy_interrupt_id = CLDMA0_INT;
1080         }
1081
1082         hw_info->ap_ao_base = t7xx_pcie_addr_transfer(pbase->pcie_ext_reg_base,
1083                                                       pbase->pcie_dev_reg_trsl_addr, phy_ao_base);
1084         hw_info->ap_pdn_base = t7xx_pcie_addr_transfer(pbase->pcie_ext_reg_base,
1085                                                        pbase->pcie_dev_reg_trsl_addr, phy_pd_base);
1086 }
1087
1088 static int t7xx_cldma_default_recv_skb(struct cldma_queue *queue, struct sk_buff *skb)
1089 {
1090         dev_kfree_skb_any(skb);
1091         return 0;
1092 }
1093
1094 int t7xx_cldma_alloc(enum cldma_id hif_id, struct t7xx_pci_dev *t7xx_dev)
1095 {
1096         struct device *dev = &t7xx_dev->pdev->dev;
1097         struct cldma_ctrl *md_ctrl;
1098
1099         md_ctrl = devm_kzalloc(dev, sizeof(*md_ctrl), GFP_KERNEL);
1100         if (!md_ctrl)
1101                 return -ENOMEM;
1102
1103         md_ctrl->t7xx_dev = t7xx_dev;
1104         md_ctrl->dev = dev;
1105         md_ctrl->hif_id = hif_id;
1106         md_ctrl->recv_skb = t7xx_cldma_default_recv_skb;
1107         t7xx_hw_info_init(md_ctrl);
1108         t7xx_dev->md->md_ctrl[hif_id] = md_ctrl;
1109         return 0;
1110 }
1111
1112 static void t7xx_cldma_resume_early(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1113 {
1114         struct cldma_ctrl *md_ctrl = entity_param;
1115         struct t7xx_cldma_hw *hw_info;
1116         unsigned long flags;
1117         int qno_t;
1118
1119         hw_info = &md_ctrl->hw_info;
1120
1121         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1122         t7xx_cldma_hw_restore(hw_info);
1123         for (qno_t = 0; qno_t < CLDMA_TXQ_NUM; qno_t++) {
1124                 t7xx_cldma_hw_set_start_addr(hw_info, qno_t, md_ctrl->txq[qno_t].tx_next->gpd_addr,
1125                                              MTK_TX);
1126                 t7xx_cldma_hw_set_start_addr(hw_info, qno_t, md_ctrl->rxq[qno_t].tr_done->gpd_addr,
1127                                              MTK_RX);
1128         }
1129         t7xx_cldma_enable_irq(md_ctrl);
1130         t7xx_cldma_hw_start_queue(hw_info, CLDMA_ALL_Q, MTK_RX);
1131         md_ctrl->rxq_active |= TXRX_STATUS_BITMASK;
1132         t7xx_cldma_hw_irq_en_eq(hw_info, CLDMA_ALL_Q, MTK_RX);
1133         t7xx_cldma_hw_irq_en_txrx(hw_info, CLDMA_ALL_Q, MTK_RX);
1134         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1135 }
1136
1137 static int t7xx_cldma_resume(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1138 {
1139         struct cldma_ctrl *md_ctrl = entity_param;
1140         unsigned long flags;
1141
1142         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1143         md_ctrl->txq_active |= TXRX_STATUS_BITMASK;
1144         t7xx_cldma_hw_irq_en_txrx(&md_ctrl->hw_info, CLDMA_ALL_Q, MTK_TX);
1145         t7xx_cldma_hw_irq_en_eq(&md_ctrl->hw_info, CLDMA_ALL_Q, MTK_TX);
1146         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1147
1148         if (md_ctrl->hif_id == CLDMA_ID_MD)
1149                 t7xx_mhccif_mask_clr(t7xx_dev, D2H_SW_INT_MASK);
1150
1151         return 0;
1152 }
1153
1154 static void t7xx_cldma_suspend_late(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1155 {
1156         struct cldma_ctrl *md_ctrl = entity_param;
1157         struct t7xx_cldma_hw *hw_info;
1158         unsigned long flags;
1159
1160         hw_info = &md_ctrl->hw_info;
1161
1162         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1163         t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, MTK_RX);
1164         t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, MTK_RX);
1165         md_ctrl->rxq_active &= ~TXRX_STATUS_BITMASK;
1166         t7xx_cldma_hw_stop_all_qs(hw_info, MTK_RX);
1167         t7xx_cldma_clear_ip_busy(hw_info);
1168         t7xx_cldma_disable_irq(md_ctrl);
1169         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1170 }
1171
1172 static int t7xx_cldma_suspend(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1173 {
1174         struct cldma_ctrl *md_ctrl = entity_param;
1175         struct t7xx_cldma_hw *hw_info;
1176         unsigned long flags;
1177
1178         if (md_ctrl->hif_id == CLDMA_ID_MD)
1179                 t7xx_mhccif_mask_set(t7xx_dev, D2H_SW_INT_MASK);
1180
1181         hw_info = &md_ctrl->hw_info;
1182
1183         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1184         t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, MTK_TX);
1185         t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, MTK_TX);
1186         md_ctrl->txq_active &= ~TXRX_STATUS_BITMASK;
1187         t7xx_cldma_hw_stop_all_qs(hw_info, MTK_TX);
1188         md_ctrl->txq_started = 0;
1189         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1190
1191         return 0;
1192 }
1193
1194 static int t7xx_cldma_pm_init(struct cldma_ctrl *md_ctrl)
1195 {
1196         md_ctrl->pm_entity = kzalloc(sizeof(*md_ctrl->pm_entity), GFP_KERNEL);
1197         if (!md_ctrl->pm_entity)
1198                 return -ENOMEM;
1199
1200         md_ctrl->pm_entity->entity_param = md_ctrl;
1201
1202         if (md_ctrl->hif_id == CLDMA_ID_MD)
1203                 md_ctrl->pm_entity->id = PM_ENTITY_ID_CTRL1;
1204         else
1205                 md_ctrl->pm_entity->id = PM_ENTITY_ID_CTRL2;
1206
1207         md_ctrl->pm_entity->suspend = t7xx_cldma_suspend;
1208         md_ctrl->pm_entity->suspend_late = t7xx_cldma_suspend_late;
1209         md_ctrl->pm_entity->resume = t7xx_cldma_resume;
1210         md_ctrl->pm_entity->resume_early = t7xx_cldma_resume_early;
1211
1212         return t7xx_pci_pm_entity_register(md_ctrl->t7xx_dev, md_ctrl->pm_entity);
1213 }
1214
1215 static int t7xx_cldma_pm_uninit(struct cldma_ctrl *md_ctrl)
1216 {
1217         if (!md_ctrl->pm_entity)
1218                 return -EINVAL;
1219
1220         t7xx_pci_pm_entity_unregister(md_ctrl->t7xx_dev, md_ctrl->pm_entity);
1221         kfree(md_ctrl->pm_entity);
1222         md_ctrl->pm_entity = NULL;
1223         return 0;
1224 }
1225
1226 void t7xx_cldma_hif_hw_init(struct cldma_ctrl *md_ctrl)
1227 {
1228         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1229         unsigned long flags;
1230
1231         spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1232         t7xx_cldma_hw_stop(hw_info, MTK_TX);
1233         t7xx_cldma_hw_stop(hw_info, MTK_RX);
1234         t7xx_cldma_hw_rx_done(hw_info, EMPTY_STATUS_BITMASK | TXRX_STATUS_BITMASK);
1235         t7xx_cldma_hw_tx_done(hw_info, EMPTY_STATUS_BITMASK | TXRX_STATUS_BITMASK);
1236         t7xx_cldma_hw_init(hw_info);
1237         spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1238 }
1239
1240 static irqreturn_t t7xx_cldma_isr_handler(int irq, void *data)
1241 {
1242         struct cldma_ctrl *md_ctrl = data;
1243         u32 interrupt;
1244
1245         interrupt = md_ctrl->hw_info.phy_interrupt_id;
1246         t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, interrupt);
1247         t7xx_cldma_irq_work_cb(md_ctrl);
1248         t7xx_pcie_mac_clear_int_status(md_ctrl->t7xx_dev, interrupt);
1249         t7xx_pcie_mac_set_int(md_ctrl->t7xx_dev, interrupt);
1250         return IRQ_HANDLED;
1251 }
1252
1253 static void t7xx_cldma_destroy_wqs(struct cldma_ctrl *md_ctrl)
1254 {
1255         int i;
1256
1257         for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1258                 if (md_ctrl->txq[i].worker) {
1259                         destroy_workqueue(md_ctrl->txq[i].worker);
1260                         md_ctrl->txq[i].worker = NULL;
1261                 }
1262         }
1263
1264         for (i = 0; i < CLDMA_RXQ_NUM; i++) {
1265                 if (md_ctrl->rxq[i].worker) {
1266                         destroy_workqueue(md_ctrl->rxq[i].worker);
1267                         md_ctrl->rxq[i].worker = NULL;
1268                 }
1269         }
1270 }
1271
1272 /**
1273  * t7xx_cldma_init() - Initialize CLDMA.
1274  * @md_ctrl: CLDMA context structure.
1275  *
1276  * Allocate and initialize device power management entity.
1277  * Initialize HIF TX/RX queue structure.
1278  * Register CLDMA callback ISR with PCIe driver.
1279  *
1280  * Return:
1281  * * 0          - Success.
1282  * * -ERROR     - Error code from failure sub-initializations.
1283  */
1284 int t7xx_cldma_init(struct cldma_ctrl *md_ctrl)
1285 {
1286         struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1287         int ret, i;
1288
1289         md_ctrl->txq_active = 0;
1290         md_ctrl->rxq_active = 0;
1291         md_ctrl->is_late_init = false;
1292
1293         ret = t7xx_cldma_pm_init(md_ctrl);
1294         if (ret)
1295                 return ret;
1296
1297         spin_lock_init(&md_ctrl->cldma_lock);
1298
1299         for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1300                 md_cd_queue_struct_init(&md_ctrl->txq[i], md_ctrl, MTK_TX, i);
1301                 md_ctrl->txq[i].worker =
1302                         alloc_ordered_workqueue("md_hif%d_tx%d_worker",
1303                                         WQ_MEM_RECLAIM | (i ? 0 : WQ_HIGHPRI),
1304                                         md_ctrl->hif_id, i);
1305                 if (!md_ctrl->txq[i].worker)
1306                         goto err_workqueue;
1307
1308                 INIT_WORK(&md_ctrl->txq[i].cldma_work, t7xx_cldma_tx_done);
1309         }
1310
1311         for (i = 0; i < CLDMA_RXQ_NUM; i++) {
1312                 md_cd_queue_struct_init(&md_ctrl->rxq[i], md_ctrl, MTK_RX, i);
1313                 INIT_WORK(&md_ctrl->rxq[i].cldma_work, t7xx_cldma_rx_done);
1314
1315                 md_ctrl->rxq[i].worker =
1316                         alloc_ordered_workqueue("md_hif%d_rx%d_worker",
1317                                                 WQ_MEM_RECLAIM,
1318                                                 md_ctrl->hif_id, i);
1319                 if (!md_ctrl->rxq[i].worker)
1320                         goto err_workqueue;
1321         }
1322
1323         t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, hw_info->phy_interrupt_id);
1324         md_ctrl->t7xx_dev->intr_handler[hw_info->phy_interrupt_id] = t7xx_cldma_isr_handler;
1325         md_ctrl->t7xx_dev->intr_thread[hw_info->phy_interrupt_id] = NULL;
1326         md_ctrl->t7xx_dev->callback_param[hw_info->phy_interrupt_id] = md_ctrl;
1327         t7xx_pcie_mac_clear_int_status(md_ctrl->t7xx_dev, hw_info->phy_interrupt_id);
1328         return 0;
1329
1330 err_workqueue:
1331         t7xx_cldma_destroy_wqs(md_ctrl);
1332         t7xx_cldma_pm_uninit(md_ctrl);
1333         return -ENOMEM;
1334 }
1335
1336 void t7xx_cldma_switch_cfg(struct cldma_ctrl *md_ctrl)
1337 {
1338         t7xx_cldma_late_release(md_ctrl);
1339         t7xx_cldma_late_init(md_ctrl);
1340 }
1341
1342 void t7xx_cldma_exit(struct cldma_ctrl *md_ctrl)
1343 {
1344         t7xx_cldma_stop(md_ctrl);
1345         t7xx_cldma_late_release(md_ctrl);
1346         t7xx_cldma_destroy_wqs(md_ctrl);
1347         t7xx_cldma_pm_uninit(md_ctrl);
1348 }