2 * Copyright (c) 2015-2016 Quantenna Communications, Inc.
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
17 #include <linux/kernel.h>
18 #include <linux/module.h>
19 #include <linux/firmware.h>
20 #include <linux/pci.h>
21 #include <linux/vmalloc.h>
22 #include <linux/delay.h>
23 #include <linux/interrupt.h>
24 #include <linux/sched.h>
25 #include <linux/completion.h>
26 #include <linux/crc32.h>
27 #include <linux/spinlock.h>
28 #include <linux/circ_buf.h>
29 #include <linux/log2.h>
31 #include "qtn_hw_ids.h"
32 #include "pcie_bus_priv.h"
37 static bool use_msi = true;
38 module_param(use_msi, bool, 0644);
39 MODULE_PARM_DESC(use_msi, "set 0 to use legacy interrupt");
41 static unsigned int tx_bd_size_param = 32;
42 module_param(tx_bd_size_param, uint, 0644);
43 MODULE_PARM_DESC(tx_bd_size_param, "Tx descriptors queue size, power of two");
45 static unsigned int rx_bd_size_param = 256;
46 module_param(rx_bd_size_param, uint, 0644);
47 MODULE_PARM_DESC(rx_bd_size_param, "Rx descriptors queue size, power of two");
49 static u8 flashboot = 1;
50 module_param(flashboot, byte, 0644);
51 MODULE_PARM_DESC(flashboot, "set to 0 to use FW binary file on FS");
53 #define DRV_NAME "qtnfmac_pearl_pcie"
55 static inline void qtnf_non_posted_write(u32 val, void __iomem *basereg)
59 /* flush posted write */
63 static inline void qtnf_init_hdp_irqs(struct qtnf_pcie_bus_priv *priv)
67 spin_lock_irqsave(&priv->irq_lock, flags);
68 priv->pcie_irq_mask = (PCIE_HDP_INT_RX_BITS | PCIE_HDP_INT_TX_BITS);
69 spin_unlock_irqrestore(&priv->irq_lock, flags);
72 static inline void qtnf_enable_hdp_irqs(struct qtnf_pcie_bus_priv *priv)
76 spin_lock_irqsave(&priv->irq_lock, flags);
77 writel(priv->pcie_irq_mask, PCIE_HDP_INT_EN(priv->pcie_reg_base));
78 spin_unlock_irqrestore(&priv->irq_lock, flags);
81 static inline void qtnf_disable_hdp_irqs(struct qtnf_pcie_bus_priv *priv)
85 spin_lock_irqsave(&priv->irq_lock, flags);
86 writel(0x0, PCIE_HDP_INT_EN(priv->pcie_reg_base));
87 spin_unlock_irqrestore(&priv->irq_lock, flags);
90 static inline void qtnf_en_rxdone_irq(struct qtnf_pcie_bus_priv *priv)
94 spin_lock_irqsave(&priv->irq_lock, flags);
95 priv->pcie_irq_mask |= PCIE_HDP_INT_RX_BITS;
96 writel(priv->pcie_irq_mask, PCIE_HDP_INT_EN(priv->pcie_reg_base));
97 spin_unlock_irqrestore(&priv->irq_lock, flags);
100 static inline void qtnf_dis_rxdone_irq(struct qtnf_pcie_bus_priv *priv)
104 spin_lock_irqsave(&priv->irq_lock, flags);
105 priv->pcie_irq_mask &= ~PCIE_HDP_INT_RX_BITS;
106 writel(priv->pcie_irq_mask, PCIE_HDP_INT_EN(priv->pcie_reg_base));
107 spin_unlock_irqrestore(&priv->irq_lock, flags);
110 static inline void qtnf_en_txdone_irq(struct qtnf_pcie_bus_priv *priv)
114 spin_lock_irqsave(&priv->irq_lock, flags);
115 priv->pcie_irq_mask |= PCIE_HDP_INT_TX_BITS;
116 writel(priv->pcie_irq_mask, PCIE_HDP_INT_EN(priv->pcie_reg_base));
117 spin_unlock_irqrestore(&priv->irq_lock, flags);
120 static inline void qtnf_dis_txdone_irq(struct qtnf_pcie_bus_priv *priv)
124 spin_lock_irqsave(&priv->irq_lock, flags);
125 priv->pcie_irq_mask &= ~PCIE_HDP_INT_TX_BITS;
126 writel(priv->pcie_irq_mask, PCIE_HDP_INT_EN(priv->pcie_reg_base));
127 spin_unlock_irqrestore(&priv->irq_lock, flags);
130 static int qtnf_pcie_init_irq(struct qtnf_pcie_bus_priv *priv)
132 struct pci_dev *pdev = priv->pdev;
134 /* fall back to legacy INTx interrupts by default */
135 priv->msi_enabled = 0;
137 /* check if MSI capability is available */
139 if (!pci_enable_msi(pdev)) {
140 pr_debug("MSI interrupt enabled\n");
141 priv->msi_enabled = 1;
143 pr_warn("failed to enable MSI interrupts");
147 if (!priv->msi_enabled) {
148 pr_warn("legacy PCIE interrupts enabled\n");
155 static void qtnf_deassert_intx(struct qtnf_pcie_bus_priv *priv)
157 void __iomem *reg = priv->sysctl_bar + PEARL_PCIE_CFG0_OFFSET;
161 cfg &= ~PEARL_ASSERT_INTX;
162 qtnf_non_posted_write(cfg, reg);
165 static void qtnf_ipc_gen_ep_int(void *arg)
167 const struct qtnf_pcie_bus_priv *priv = arg;
168 const u32 data = QTN_PEARL_IPC_IRQ_WORD(QTN_PEARL_LHOST_IPC_IRQ);
169 void __iomem *reg = priv->sysctl_bar +
170 QTN_PEARL_SYSCTL_LHOST_IRQ_OFFSET;
172 qtnf_non_posted_write(data, reg);
175 static void __iomem *qtnf_map_bar(struct qtnf_pcie_bus_priv *priv, u8 index)
182 ret = pcim_iomap_regions(priv->pdev, 1 << index, DRV_NAME);
184 return IOMEM_ERR_PTR(ret);
186 busaddr = pci_resource_start(priv->pdev, index);
187 len = pci_resource_len(priv->pdev, index);
188 vaddr = pcim_iomap_table(priv->pdev)[index];
190 return IOMEM_ERR_PTR(-ENOMEM);
192 pr_debug("BAR%u vaddr=0x%p busaddr=%pad len=%u\n",
193 index, vaddr, &busaddr, (int)len);
198 static void qtnf_pcie_control_rx_callback(void *arg, const u8 *buf, size_t len)
200 struct qtnf_pcie_bus_priv *priv = arg;
201 struct qtnf_bus *bus = pci_get_drvdata(priv->pdev);
204 if (unlikely(len == 0)) {
205 pr_warn("zero length packet received\n");
209 skb = __dev_alloc_skb(len, GFP_KERNEL);
211 if (unlikely(!skb)) {
212 pr_err("failed to allocate skb\n");
216 skb_put_data(skb, buf, len);
218 qtnf_trans_handle_rx_ctl_packet(bus, skb);
221 static int qtnf_pcie_init_shm_ipc(struct qtnf_pcie_bus_priv *priv)
223 struct qtnf_shm_ipc_region __iomem *ipc_tx_reg;
224 struct qtnf_shm_ipc_region __iomem *ipc_rx_reg;
225 const struct qtnf_shm_ipc_int ipc_int = { qtnf_ipc_gen_ep_int, priv };
226 const struct qtnf_shm_ipc_rx_callback rx_callback = {
227 qtnf_pcie_control_rx_callback, priv };
229 ipc_tx_reg = &priv->bda->bda_shm_reg1;
230 ipc_rx_reg = &priv->bda->bda_shm_reg2;
232 qtnf_shm_ipc_init(&priv->shm_ipc_ep_in, QTNF_SHM_IPC_OUTBOUND,
233 ipc_tx_reg, priv->workqueue,
234 &ipc_int, &rx_callback);
235 qtnf_shm_ipc_init(&priv->shm_ipc_ep_out, QTNF_SHM_IPC_INBOUND,
236 ipc_rx_reg, priv->workqueue,
237 &ipc_int, &rx_callback);
242 static void qtnf_pcie_free_shm_ipc(struct qtnf_pcie_bus_priv *priv)
244 qtnf_shm_ipc_free(&priv->shm_ipc_ep_in);
245 qtnf_shm_ipc_free(&priv->shm_ipc_ep_out);
248 static int qtnf_pcie_init_memory(struct qtnf_pcie_bus_priv *priv)
252 priv->sysctl_bar = qtnf_map_bar(priv, QTN_SYSCTL_BAR);
253 if (IS_ERR(priv->sysctl_bar)) {
254 pr_err("failed to map BAR%u\n", QTN_SYSCTL_BAR);
258 priv->dmareg_bar = qtnf_map_bar(priv, QTN_DMA_BAR);
259 if (IS_ERR(priv->dmareg_bar)) {
260 pr_err("failed to map BAR%u\n", QTN_DMA_BAR);
264 priv->epmem_bar = qtnf_map_bar(priv, QTN_SHMEM_BAR);
265 if (IS_ERR(priv->epmem_bar)) {
266 pr_err("failed to map BAR%u\n", QTN_SHMEM_BAR);
270 priv->pcie_reg_base = priv->dmareg_bar;
271 priv->bda = priv->epmem_bar;
272 writel(priv->msi_enabled, &priv->bda->bda_rc_msi_enabled);
277 static void qtnf_tune_pcie_mps(struct qtnf_pcie_bus_priv *priv)
279 struct pci_dev *pdev = priv->pdev;
280 struct pci_dev *parent;
281 int mps_p, mps_o, mps_m, mps;
285 mps_o = pcie_get_mps(pdev);
287 /* maximum supported mps */
288 mps_m = 128 << pdev->pcie_mpss;
290 /* suggested new mps value */
293 if (pdev->bus && pdev->bus->self) {
294 /* parent (bus) mps */
295 parent = pdev->bus->self;
297 if (pci_is_pcie(parent)) {
298 mps_p = pcie_get_mps(parent);
299 mps = min(mps_m, mps_p);
303 ret = pcie_set_mps(pdev, mps);
305 pr_err("failed to set mps to %d, keep using current %d\n",
311 pr_debug("set mps to %d (was %d, max %d)\n", mps, mps_o, mps_m);
315 static int qtnf_is_state(__le32 __iomem *reg, u32 state)
322 static void qtnf_set_state(__le32 __iomem *reg, u32 state)
326 qtnf_non_posted_write(state | s, reg);
329 static void qtnf_clear_state(__le32 __iomem *reg, u32 state)
333 qtnf_non_posted_write(s & ~state, reg);
336 static int qtnf_poll_state(__le32 __iomem *reg, u32 state, u32 delay_in_ms)
340 while ((qtnf_is_state(reg, state) == 0)) {
341 usleep_range(1000, 1200);
342 if (++timeout > delay_in_ms)
349 static int alloc_skb_array(struct qtnf_pcie_bus_priv *priv)
351 struct sk_buff **vaddr;
354 len = priv->tx_bd_num * sizeof(*priv->tx_skb) +
355 priv->rx_bd_num * sizeof(*priv->rx_skb);
356 vaddr = devm_kzalloc(&priv->pdev->dev, len, GFP_KERNEL);
361 priv->tx_skb = vaddr;
363 vaddr += priv->tx_bd_num;
364 priv->rx_skb = vaddr;
369 static int alloc_bd_table(struct qtnf_pcie_bus_priv *priv)
375 len = priv->tx_bd_num * sizeof(struct qtnf_tx_bd) +
376 priv->rx_bd_num * sizeof(struct qtnf_rx_bd);
378 vaddr = dmam_alloc_coherent(&priv->pdev->dev, len, &paddr, GFP_KERNEL);
384 memset(vaddr, 0, len);
386 priv->bd_table_vaddr = vaddr;
387 priv->bd_table_paddr = paddr;
388 priv->bd_table_len = len;
390 priv->tx_bd_vbase = vaddr;
391 priv->tx_bd_pbase = paddr;
393 pr_debug("TX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr);
395 priv->tx_bd_r_index = 0;
396 priv->tx_bd_w_index = 0;
400 vaddr = ((struct qtnf_tx_bd *)vaddr) + priv->tx_bd_num;
401 paddr += priv->tx_bd_num * sizeof(struct qtnf_tx_bd);
403 priv->rx_bd_vbase = vaddr;
404 priv->rx_bd_pbase = paddr;
406 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
407 writel(QTN_HOST_HI32(paddr),
408 PCIE_HDP_TX_HOST_Q_BASE_H(priv->pcie_reg_base));
410 writel(QTN_HOST_LO32(paddr),
411 PCIE_HDP_TX_HOST_Q_BASE_L(priv->pcie_reg_base));
412 writel(priv->rx_bd_num | (sizeof(struct qtnf_rx_bd)) << 16,
413 PCIE_HDP_TX_HOST_Q_SZ_CTRL(priv->pcie_reg_base));
415 pr_debug("RX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr);
420 static int skb2rbd_attach(struct qtnf_pcie_bus_priv *priv, u16 index)
422 struct qtnf_rx_bd *rxbd;
426 skb = __netdev_alloc_skb_ip_align(NULL, SKB_BUF_SIZE, GFP_ATOMIC);
428 priv->rx_skb[index] = NULL;
432 priv->rx_skb[index] = skb;
433 rxbd = &priv->rx_bd_vbase[index];
435 paddr = pci_map_single(priv->pdev, skb->data,
436 SKB_BUF_SIZE, PCI_DMA_FROMDEVICE);
437 if (pci_dma_mapping_error(priv->pdev, paddr)) {
438 pr_err("skb DMA mapping error: %pad\n", &paddr);
442 /* keep rx skb paddrs in rx buffer descriptors for cleanup purposes */
443 rxbd->addr = cpu_to_le32(QTN_HOST_LO32(paddr));
444 rxbd->addr_h = cpu_to_le32(QTN_HOST_HI32(paddr));
447 priv->rx_bd_w_index = index;
449 /* sync up all descriptor updates */
452 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
453 writel(QTN_HOST_HI32(paddr),
454 PCIE_HDP_HHBM_BUF_PTR_H(priv->pcie_reg_base));
456 writel(QTN_HOST_LO32(paddr),
457 PCIE_HDP_HHBM_BUF_PTR(priv->pcie_reg_base));
459 writel(index, PCIE_HDP_TX_HOST_Q_WR_PTR(priv->pcie_reg_base));
463 static int alloc_rx_buffers(struct qtnf_pcie_bus_priv *priv)
468 memset(priv->rx_bd_vbase, 0x0,
469 priv->rx_bd_num * sizeof(struct qtnf_rx_bd));
471 for (i = 0; i < priv->rx_bd_num; i++) {
472 ret = skb2rbd_attach(priv, i);
480 /* all rx/tx activity should have ceased before calling this function */
481 static void free_xfer_buffers(void *data)
483 struct qtnf_pcie_bus_priv *priv = (struct qtnf_pcie_bus_priv *)data;
484 struct qtnf_rx_bd *rxbd;
488 /* free rx buffers */
489 for (i = 0; i < priv->rx_bd_num; i++) {
490 if (priv->rx_skb && priv->rx_skb[i]) {
491 rxbd = &priv->rx_bd_vbase[i];
492 paddr = QTN_HOST_ADDR(le32_to_cpu(rxbd->addr_h),
493 le32_to_cpu(rxbd->addr));
494 pci_unmap_single(priv->pdev, paddr, SKB_BUF_SIZE,
497 dev_kfree_skb_any(priv->rx_skb[i]);
501 /* free tx buffers */
502 for (i = 0; i < priv->tx_bd_num; i++) {
503 if (priv->tx_skb && priv->tx_skb[i]) {
504 dev_kfree_skb_any(priv->tx_skb[i]);
505 priv->tx_skb[i] = NULL;
510 static int qtnf_hhbm_init(struct qtnf_pcie_bus_priv *priv)
514 val = readl(PCIE_HHBM_CONFIG(priv->pcie_reg_base));
515 val |= HHBM_CONFIG_SOFT_RESET;
516 writel(val, PCIE_HHBM_CONFIG(priv->pcie_reg_base));
517 usleep_range(50, 100);
518 val &= ~HHBM_CONFIG_SOFT_RESET;
519 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
522 writel(val, PCIE_HHBM_CONFIG(priv->pcie_reg_base));
523 writel(priv->rx_bd_num, PCIE_HHBM_Q_LIMIT_REG(priv->pcie_reg_base));
528 static int qtnf_pcie_init_xfer(struct qtnf_pcie_bus_priv *priv)
533 priv->tx_bd_num = tx_bd_size_param;
534 priv->rx_bd_num = rx_bd_size_param;
535 priv->rx_bd_w_index = 0;
536 priv->rx_bd_r_index = 0;
538 if (!priv->tx_bd_num || !is_power_of_2(priv->tx_bd_num)) {
539 pr_err("tx_bd_size_param %u is not power of two\n",
544 val = priv->tx_bd_num * sizeof(struct qtnf_tx_bd);
545 if (val > PCIE_HHBM_MAX_SIZE) {
546 pr_err("tx_bd_size_param %u is too large\n",
551 if (!priv->rx_bd_num || !is_power_of_2(priv->rx_bd_num)) {
552 pr_err("rx_bd_size_param %u is not power of two\n",
557 val = priv->rx_bd_num * sizeof(dma_addr_t);
558 if (val > PCIE_HHBM_MAX_SIZE) {
559 pr_err("rx_bd_size_param %u is too large\n",
564 ret = qtnf_hhbm_init(priv);
566 pr_err("failed to init h/w queues\n");
570 ret = alloc_skb_array(priv);
572 pr_err("failed to allocate skb array\n");
576 ret = alloc_bd_table(priv);
578 pr_err("failed to allocate bd table\n");
582 ret = alloc_rx_buffers(priv);
584 pr_err("failed to allocate rx buffers\n");
591 static void qtnf_pcie_data_tx_reclaim(struct qtnf_pcie_bus_priv *priv)
593 struct qtnf_tx_bd *txbd;
601 spin_lock_irqsave(&priv->tx_reclaim_lock, flags);
603 tx_done_index = readl(PCIE_HDP_RX0DMA_CNT(priv->pcie_reg_base))
604 & (priv->tx_bd_num - 1);
606 i = priv->tx_bd_r_index;
608 while (CIRC_CNT(tx_done_index, i, priv->tx_bd_num)) {
609 skb = priv->tx_skb[i];
611 txbd = &priv->tx_bd_vbase[i];
612 paddr = QTN_HOST_ADDR(le32_to_cpu(txbd->addr_h),
613 le32_to_cpu(txbd->addr));
614 pci_unmap_single(priv->pdev, paddr, skb->len,
618 skb->dev->stats.tx_packets++;
619 skb->dev->stats.tx_bytes += skb->len;
621 if (netif_queue_stopped(skb->dev))
622 netif_wake_queue(skb->dev);
625 dev_kfree_skb_any(skb);
628 priv->tx_skb[i] = NULL;
631 if (++i >= priv->tx_bd_num)
635 priv->tx_reclaim_done += count;
636 priv->tx_reclaim_req++;
637 priv->tx_bd_r_index = i;
639 spin_unlock_irqrestore(&priv->tx_reclaim_lock, flags);
642 static int qtnf_tx_queue_ready(struct qtnf_pcie_bus_priv *priv)
644 if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index,
646 qtnf_pcie_data_tx_reclaim(priv);
648 if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index,
650 pr_warn_ratelimited("reclaim full Tx queue\n");
651 priv->tx_full_count++;
659 static int qtnf_pcie_data_tx(struct qtnf_bus *bus, struct sk_buff *skb)
661 struct qtnf_pcie_bus_priv *priv = (void *)get_bus_priv(bus);
662 dma_addr_t txbd_paddr, skb_paddr;
663 struct qtnf_tx_bd *txbd;
669 spin_lock_irqsave(&priv->tx0_lock, flags);
671 if (!qtnf_tx_queue_ready(priv)) {
673 netif_stop_queue(skb->dev);
675 spin_unlock_irqrestore(&priv->tx0_lock, flags);
676 return NETDEV_TX_BUSY;
679 i = priv->tx_bd_w_index;
680 priv->tx_skb[i] = skb;
683 skb_paddr = pci_map_single(priv->pdev, skb->data,
684 skb->len, PCI_DMA_TODEVICE);
685 if (pci_dma_mapping_error(priv->pdev, skb_paddr)) {
686 pr_err("skb DMA mapping error: %pad\n", &skb_paddr);
691 txbd = &priv->tx_bd_vbase[i];
692 txbd->addr = cpu_to_le32(QTN_HOST_LO32(skb_paddr));
693 txbd->addr_h = cpu_to_le32(QTN_HOST_HI32(skb_paddr));
695 info = (len & QTN_PCIE_TX_DESC_LEN_MASK) << QTN_PCIE_TX_DESC_LEN_SHIFT;
696 txbd->info = cpu_to_le32(info);
698 /* sync up all descriptor updates before passing them to EP */
701 /* write new TX descriptor to PCIE_RX_FIFO on EP */
702 txbd_paddr = priv->tx_bd_pbase + i * sizeof(struct qtnf_tx_bd);
704 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
705 writel(QTN_HOST_HI32(txbd_paddr),
706 PCIE_HDP_HOST_WR_DESC0_H(priv->pcie_reg_base));
708 writel(QTN_HOST_LO32(txbd_paddr),
709 PCIE_HDP_HOST_WR_DESC0(priv->pcie_reg_base));
711 if (++i >= priv->tx_bd_num)
714 priv->tx_bd_w_index = i;
718 pr_err_ratelimited("drop skb\n");
720 skb->dev->stats.tx_dropped++;
721 dev_kfree_skb_any(skb);
724 priv->tx_done_count++;
725 spin_unlock_irqrestore(&priv->tx0_lock, flags);
727 qtnf_pcie_data_tx_reclaim(priv);
732 static int qtnf_pcie_control_tx(struct qtnf_bus *bus, struct sk_buff *skb)
734 struct qtnf_pcie_bus_priv *priv = (void *)get_bus_priv(bus);
736 return qtnf_shm_ipc_send(&priv->shm_ipc_ep_in, skb->data, skb->len);
739 static irqreturn_t qtnf_interrupt(int irq, void *data)
741 struct qtnf_bus *bus = (struct qtnf_bus *)data;
742 struct qtnf_pcie_bus_priv *priv = (void *)get_bus_priv(bus);
745 priv->pcie_irq_count++;
746 status = readl(PCIE_HDP_INT_STATUS(priv->pcie_reg_base));
748 qtnf_shm_ipc_irq_handler(&priv->shm_ipc_ep_in);
749 qtnf_shm_ipc_irq_handler(&priv->shm_ipc_ep_out);
751 if (!(status & priv->pcie_irq_mask))
754 if (status & PCIE_HDP_INT_RX_BITS)
755 priv->pcie_irq_rx_count++;
757 if (status & PCIE_HDP_INT_TX_BITS)
758 priv->pcie_irq_tx_count++;
760 if (status & PCIE_HDP_INT_HHBM_UF)
761 priv->pcie_irq_uf_count++;
763 if (status & PCIE_HDP_INT_RX_BITS) {
764 qtnf_dis_rxdone_irq(priv);
765 napi_schedule(&bus->mux_napi);
768 if (status & PCIE_HDP_INT_TX_BITS) {
769 qtnf_dis_txdone_irq(priv);
770 tasklet_hi_schedule(&priv->reclaim_tq);
774 /* H/W workaround: clean all bits, not only enabled */
775 qtnf_non_posted_write(~0U, PCIE_HDP_INT_STATUS(priv->pcie_reg_base));
777 if (!priv->msi_enabled)
778 qtnf_deassert_intx(priv);
783 static int qtnf_rx_data_ready(struct qtnf_pcie_bus_priv *priv)
785 u16 index = priv->rx_bd_r_index;
786 struct qtnf_rx_bd *rxbd;
789 rxbd = &priv->rx_bd_vbase[index];
790 descw = le32_to_cpu(rxbd->info);
792 if (descw & QTN_TXDONE_MASK)
798 static int qtnf_rx_poll(struct napi_struct *napi, int budget)
800 struct qtnf_bus *bus = container_of(napi, struct qtnf_bus, mux_napi);
801 struct qtnf_pcie_bus_priv *priv = (void *)get_bus_priv(bus);
802 struct net_device *ndev = NULL;
803 struct sk_buff *skb = NULL;
805 struct qtnf_rx_bd *rxbd;
806 dma_addr_t skb_paddr;
814 while (processed < budget) {
817 if (!qtnf_rx_data_ready(priv))
820 r_idx = priv->rx_bd_r_index;
821 rxbd = &priv->rx_bd_vbase[r_idx];
822 descw = le32_to_cpu(rxbd->info);
824 skb = priv->rx_skb[r_idx];
825 psize = QTN_GET_LEN(descw);
828 if (!(descw & QTN_TXDONE_MASK)) {
829 pr_warn("skip invalid rxbd[%d]\n", r_idx);
834 pr_warn("skip missing rx_skb[%d]\n", r_idx);
838 if (skb && (skb_tailroom(skb) < psize)) {
839 pr_err("skip packet with invalid length: %u > %u\n",
840 psize, skb_tailroom(skb));
845 skb_paddr = QTN_HOST_ADDR(le32_to_cpu(rxbd->addr_h),
846 le32_to_cpu(rxbd->addr));
847 pci_unmap_single(priv->pdev, skb_paddr, SKB_BUF_SIZE,
853 ndev = qtnf_classify_skb(bus, skb);
855 ndev->stats.rx_packets++;
856 ndev->stats.rx_bytes += skb->len;
858 skb->protocol = eth_type_trans(skb, ndev);
859 napi_gro_receive(napi, skb);
861 pr_debug("drop untagged skb\n");
862 bus->mux_dev.stats.rx_dropped++;
863 dev_kfree_skb_any(skb);
867 bus->mux_dev.stats.rx_dropped++;
868 dev_kfree_skb_any(skb);
872 priv->rx_skb[r_idx] = NULL;
873 if (++r_idx >= priv->rx_bd_num)
876 priv->rx_bd_r_index = r_idx;
878 /* repalce processed buffer by a new one */
879 w_idx = priv->rx_bd_w_index;
880 while (CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index,
881 priv->rx_bd_num) > 0) {
882 if (++w_idx >= priv->rx_bd_num)
885 ret = skb2rbd_attach(priv, w_idx);
887 pr_err("failed to allocate new rx_skb[%d]\n",
897 if (processed < budget) {
899 qtnf_en_rxdone_irq(priv);
906 qtnf_pcie_data_tx_timeout(struct qtnf_bus *bus, struct net_device *ndev)
908 struct qtnf_pcie_bus_priv *priv = (void *)get_bus_priv(bus);
910 tasklet_hi_schedule(&priv->reclaim_tq);
913 static void qtnf_pcie_data_rx_start(struct qtnf_bus *bus)
915 struct qtnf_pcie_bus_priv *priv = (void *)get_bus_priv(bus);
917 qtnf_enable_hdp_irqs(priv);
918 napi_enable(&bus->mux_napi);
921 static void qtnf_pcie_data_rx_stop(struct qtnf_bus *bus)
923 struct qtnf_pcie_bus_priv *priv = (void *)get_bus_priv(bus);
925 napi_disable(&bus->mux_napi);
926 qtnf_disable_hdp_irqs(priv);
929 static const struct qtnf_bus_ops qtnf_pcie_bus_ops = {
930 /* control path methods */
931 .control_tx = qtnf_pcie_control_tx,
933 /* data path methods */
934 .data_tx = qtnf_pcie_data_tx,
935 .data_tx_timeout = qtnf_pcie_data_tx_timeout,
936 .data_rx_start = qtnf_pcie_data_rx_start,
937 .data_rx_stop = qtnf_pcie_data_rx_stop,
940 static int qtnf_ep_fw_send(struct qtnf_pcie_bus_priv *priv, uint32_t size,
941 int blk, const u8 *pblk, const u8 *fw)
943 struct pci_dev *pdev = priv->pdev;
944 struct qtnf_bus *bus = pci_get_drvdata(pdev);
946 struct qtnf_pcie_fw_hdr *hdr;
949 int hds = sizeof(*hdr);
950 struct sk_buff *skb = NULL;
954 skb = __dev_alloc_skb(QTN_PCIE_FW_BUFSZ, GFP_KERNEL);
958 skb->len = QTN_PCIE_FW_BUFSZ;
961 hdr = (struct qtnf_pcie_fw_hdr *)skb->data;
962 memcpy(hdr->boardflg, QTN_PCIE_BOARDFLG, strlen(QTN_PCIE_BOARDFLG));
963 hdr->fwsize = cpu_to_le32(size);
964 hdr->seqnum = cpu_to_le32(blk);
967 hdr->type = cpu_to_le32(QTN_FW_DSUB);
969 hdr->type = cpu_to_le32(QTN_FW_DBEGIN);
971 pdata = skb->data + hds;
973 len = QTN_PCIE_FW_BUFSZ - hds;
974 if (pblk >= (fw + size - len)) {
975 len = fw + size - pblk;
976 hdr->type = cpu_to_le32(QTN_FW_DEND);
979 hdr->pktlen = cpu_to_le32(len);
980 memcpy(pdata, pblk, len);
981 hdr->crc = cpu_to_le32(~crc32(0, pdata, len));
983 ret = qtnf_pcie_data_tx(bus, skb);
985 return (ret == NETDEV_TX_OK) ? len : 0;
989 qtnf_ep_fw_load(struct qtnf_pcie_bus_priv *priv, const u8 *fw, u32 fw_size)
991 int blk_size = QTN_PCIE_FW_BUFSZ - sizeof(struct qtnf_pcie_fw_hdr);
992 int blk_count = fw_size / blk_size + ((fw_size % blk_size) ? 1 : 0);
998 pr_debug("FW upload started: fw_addr=0x%p size=%d\n", fw, fw_size);
1000 while (blk < blk_count) {
1001 if (++threshold > 10000) {
1002 pr_err("FW upload failed: too many retries\n");
1006 len = qtnf_ep_fw_send(priv, fw_size, blk, pblk, fw);
1010 if (!((blk + 1) & QTN_PCIE_FW_DLMASK) ||
1011 (blk == (blk_count - 1))) {
1012 qtnf_set_state(&priv->bda->bda_rc_state,
1014 if (qtnf_poll_state(&priv->bda->bda_ep_state,
1016 QTN_FW_DL_TIMEOUT_MS)) {
1017 pr_err("FW upload failed: SYNC timed out\n");
1021 qtnf_clear_state(&priv->bda->bda_ep_state,
1024 if (qtnf_is_state(&priv->bda->bda_ep_state,
1026 if (blk == (blk_count - 1)) {
1028 blk_count & QTN_PCIE_FW_DLMASK;
1030 pblk -= ((last_round - 1) *
1033 blk -= QTN_PCIE_FW_DLMASK;
1034 pblk -= QTN_PCIE_FW_DLMASK * blk_size;
1037 qtnf_clear_state(&priv->bda->bda_ep_state,
1040 pr_warn("FW upload retry: block #%d\n", blk);
1044 qtnf_pcie_data_tx_reclaim(priv);
1051 pr_debug("FW upload completed: totally sent %d blocks\n", blk);
1055 static void qtnf_firmware_load(const struct firmware *fw, void *context)
1057 struct qtnf_pcie_bus_priv *priv = (void *)context;
1058 struct pci_dev *pdev = priv->pdev;
1059 struct qtnf_bus *bus = pci_get_drvdata(pdev);
1063 pr_err("failed to get firmware %s\n", bus->fwname);
1067 ret = qtnf_ep_fw_load(priv, fw->data, fw->size);
1069 pr_err("FW upload error\n");
1073 if (qtnf_poll_state(&priv->bda->bda_ep_state, QTN_EP_FW_DONE,
1074 QTN_FW_DL_TIMEOUT_MS)) {
1075 pr_err("FW bringup timed out\n");
1079 bus->fw_state = QTNF_FW_STATE_FW_DNLD_DONE;
1080 pr_info("firmware is up and running\n");
1085 release_firmware(fw);
1087 complete(&bus->request_firmware_complete);
1090 static int qtnf_bringup_fw(struct qtnf_bus *bus)
1092 struct qtnf_pcie_bus_priv *priv = (void *)get_bus_priv(bus);
1093 struct pci_dev *pdev = priv->pdev;
1095 u32 state = QTN_RC_FW_LOADRDY | QTN_RC_FW_QLINK;
1098 state |= QTN_RC_FW_FLASHBOOT;
1100 qtnf_set_state(&priv->bda->bda_rc_state, state);
1102 if (qtnf_poll_state(&priv->bda->bda_ep_state, QTN_EP_FW_LOADRDY,
1103 QTN_FW_DL_TIMEOUT_MS)) {
1104 pr_err("card is not ready\n");
1108 qtnf_clear_state(&priv->bda->bda_ep_state, QTN_EP_FW_LOADRDY);
1111 pr_info("Booting FW from flash\n");
1113 if (!qtnf_poll_state(&priv->bda->bda_ep_state, QTN_EP_FW_DONE,
1114 QTN_FW_DL_TIMEOUT_MS))
1115 bus->fw_state = QTNF_FW_STATE_FW_DNLD_DONE;
1120 pr_info("starting firmware upload: %s\n", bus->fwname);
1122 ret = reject_firmware_nowait(THIS_MODULE, 1, bus->fwname, &pdev->dev,
1123 GFP_KERNEL, priv, qtnf_firmware_load);
1125 pr_err("request_firmware_nowait error %d\n", ret);
1132 static void qtnf_reclaim_tasklet_fn(unsigned long data)
1134 struct qtnf_pcie_bus_priv *priv = (void *)data;
1136 qtnf_pcie_data_tx_reclaim(priv);
1137 qtnf_en_txdone_irq(priv);
1140 static int qtnf_dbg_mps_show(struct seq_file *s, void *data)
1142 struct qtnf_bus *bus = dev_get_drvdata(s->private);
1143 struct qtnf_pcie_bus_priv *priv = get_bus_priv(bus);
1145 seq_printf(s, "%d\n", priv->mps);
1150 static int qtnf_dbg_msi_show(struct seq_file *s, void *data)
1152 struct qtnf_bus *bus = dev_get_drvdata(s->private);
1153 struct qtnf_pcie_bus_priv *priv = get_bus_priv(bus);
1155 seq_printf(s, "%u\n", priv->msi_enabled);
1160 static int qtnf_dbg_irq_stats(struct seq_file *s, void *data)
1162 struct qtnf_bus *bus = dev_get_drvdata(s->private);
1163 struct qtnf_pcie_bus_priv *priv = get_bus_priv(bus);
1164 u32 reg = readl(PCIE_HDP_INT_EN(priv->pcie_reg_base));
1167 seq_printf(s, "pcie_irq_count(%u)\n", priv->pcie_irq_count);
1168 seq_printf(s, "pcie_irq_tx_count(%u)\n", priv->pcie_irq_tx_count);
1169 status = reg & PCIE_HDP_INT_TX_BITS;
1170 seq_printf(s, "pcie_irq_tx_status(%s)\n",
1171 (status == PCIE_HDP_INT_TX_BITS) ? "EN" : "DIS");
1172 seq_printf(s, "pcie_irq_rx_count(%u)\n", priv->pcie_irq_rx_count);
1173 status = reg & PCIE_HDP_INT_RX_BITS;
1174 seq_printf(s, "pcie_irq_rx_status(%s)\n",
1175 (status == PCIE_HDP_INT_RX_BITS) ? "EN" : "DIS");
1176 seq_printf(s, "pcie_irq_uf_count(%u)\n", priv->pcie_irq_uf_count);
1177 status = reg & PCIE_HDP_INT_HHBM_UF;
1178 seq_printf(s, "pcie_irq_hhbm_uf_status(%s)\n",
1179 (status == PCIE_HDP_INT_HHBM_UF) ? "EN" : "DIS");
1184 static int qtnf_dbg_hdp_stats(struct seq_file *s, void *data)
1186 struct qtnf_bus *bus = dev_get_drvdata(s->private);
1187 struct qtnf_pcie_bus_priv *priv = get_bus_priv(bus);
1189 seq_printf(s, "tx_full_count(%u)\n", priv->tx_full_count);
1190 seq_printf(s, "tx_done_count(%u)\n", priv->tx_done_count);
1191 seq_printf(s, "tx_reclaim_done(%u)\n", priv->tx_reclaim_done);
1192 seq_printf(s, "tx_reclaim_req(%u)\n", priv->tx_reclaim_req);
1194 seq_printf(s, "tx_bd_r_index(%u)\n", priv->tx_bd_r_index);
1195 seq_printf(s, "tx_bd_p_index(%u)\n",
1196 readl(PCIE_HDP_RX0DMA_CNT(priv->pcie_reg_base))
1197 & (priv->tx_bd_num - 1));
1198 seq_printf(s, "tx_bd_w_index(%u)\n", priv->tx_bd_w_index);
1199 seq_printf(s, "tx queue len(%u)\n",
1200 CIRC_CNT(priv->tx_bd_w_index, priv->tx_bd_r_index,
1203 seq_printf(s, "rx_bd_r_index(%u)\n", priv->rx_bd_r_index);
1204 seq_printf(s, "rx_bd_p_index(%u)\n",
1205 readl(PCIE_HDP_TX0DMA_CNT(priv->pcie_reg_base))
1206 & (priv->rx_bd_num - 1));
1207 seq_printf(s, "rx_bd_w_index(%u)\n", priv->rx_bd_w_index);
1208 seq_printf(s, "rx alloc queue len(%u)\n",
1209 CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index,
1215 static int qtnf_dbg_shm_stats(struct seq_file *s, void *data)
1217 struct qtnf_bus *bus = dev_get_drvdata(s->private);
1218 struct qtnf_pcie_bus_priv *priv = get_bus_priv(bus);
1220 seq_printf(s, "shm_ipc_ep_in.tx_packet_count(%zu)\n",
1221 priv->shm_ipc_ep_in.tx_packet_count);
1222 seq_printf(s, "shm_ipc_ep_in.rx_packet_count(%zu)\n",
1223 priv->shm_ipc_ep_in.rx_packet_count);
1224 seq_printf(s, "shm_ipc_ep_out.tx_packet_count(%zu)\n",
1225 priv->shm_ipc_ep_out.tx_timeout_count);
1226 seq_printf(s, "shm_ipc_ep_out.rx_packet_count(%zu)\n",
1227 priv->shm_ipc_ep_out.rx_packet_count);
1232 static int qtnf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1234 struct qtnf_pcie_bus_priv *pcie_priv;
1235 struct qtnf_bus *bus;
1238 bus = devm_kzalloc(&pdev->dev,
1239 sizeof(*bus) + sizeof(*pcie_priv), GFP_KERNEL);
1245 pcie_priv = get_bus_priv(bus);
1247 pci_set_drvdata(pdev, bus);
1248 bus->bus_ops = &qtnf_pcie_bus_ops;
1249 bus->dev = &pdev->dev;
1250 bus->fw_state = QTNF_FW_STATE_RESET;
1251 pcie_priv->pdev = pdev;
1253 strcpy(bus->fwname, QTN_PCI_PEARL_FW_NAME);
1254 init_completion(&bus->request_firmware_complete);
1255 mutex_init(&bus->bus_lock);
1256 spin_lock_init(&pcie_priv->tx0_lock);
1257 spin_lock_init(&pcie_priv->irq_lock);
1258 spin_lock_init(&pcie_priv->tx_reclaim_lock);
1261 pcie_priv->tx_full_count = 0;
1262 pcie_priv->tx_done_count = 0;
1263 pcie_priv->pcie_irq_count = 0;
1264 pcie_priv->pcie_irq_rx_count = 0;
1265 pcie_priv->pcie_irq_tx_count = 0;
1266 pcie_priv->pcie_irq_uf_count = 0;
1267 pcie_priv->tx_reclaim_done = 0;
1268 pcie_priv->tx_reclaim_req = 0;
1270 pcie_priv->workqueue = create_singlethread_workqueue("QTNF_PEARL_PCIE");
1271 if (!pcie_priv->workqueue) {
1272 pr_err("failed to alloc bus workqueue\n");
1277 if (!pci_is_pcie(pdev)) {
1278 pr_err("device %s is not PCI Express\n", pci_name(pdev));
1283 qtnf_tune_pcie_mps(pcie_priv);
1285 ret = pcim_enable_device(pdev);
1287 pr_err("failed to init PCI device %x\n", pdev->device);
1290 pr_debug("successful init of PCI device %x\n", pdev->device);
1293 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
1294 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1296 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1299 pr_err("PCIE DMA coherent mask init failed\n");
1303 pcim_pin_device(pdev);
1304 pci_set_master(pdev);
1306 ret = qtnf_pcie_init_irq(pcie_priv);
1308 pr_err("irq init failed\n");
1312 ret = qtnf_pcie_init_memory(pcie_priv);
1314 pr_err("PCIE memory init failed\n");
1318 ret = qtnf_pcie_init_shm_ipc(pcie_priv);
1320 pr_err("PCIE SHM IPC init failed\n");
1324 ret = devm_add_action(&pdev->dev, free_xfer_buffers, (void *)pcie_priv);
1326 pr_err("custom release callback init failed\n");
1330 ret = qtnf_pcie_init_xfer(pcie_priv);
1332 pr_err("PCIE xfer init failed\n");
1336 /* init default irq settings */
1337 qtnf_init_hdp_irqs(pcie_priv);
1339 /* start with disabled irqs */
1340 qtnf_disable_hdp_irqs(pcie_priv);
1342 ret = devm_request_irq(&pdev->dev, pdev->irq, &qtnf_interrupt, 0,
1343 "qtnf_pcie_irq", (void *)bus);
1345 pr_err("failed to request pcie irq %d\n", pdev->irq);
1349 tasklet_init(&pcie_priv->reclaim_tq, qtnf_reclaim_tasklet_fn,
1350 (unsigned long)pcie_priv);
1351 init_dummy_netdev(&bus->mux_dev);
1352 netif_napi_add(&bus->mux_dev, &bus->mux_napi,
1355 ret = qtnf_bringup_fw(bus);
1357 goto err_bringup_fw;
1359 wait_for_completion(&bus->request_firmware_complete);
1361 if (bus->fw_state != QTNF_FW_STATE_FW_DNLD_DONE) {
1362 pr_err("failed to start FW\n");
1363 goto err_bringup_fw;
1366 if (qtnf_poll_state(&pcie_priv->bda->bda_ep_state, QTN_EP_FW_QLINK_DONE,
1367 QTN_FW_QLINK_TIMEOUT_MS)) {
1368 pr_err("FW runtime failure\n");
1369 goto err_bringup_fw;
1372 ret = qtnf_core_attach(bus);
1374 pr_err("failed to attach core\n");
1375 goto err_bringup_fw;
1378 qtnf_debugfs_init(bus, DRV_NAME);
1379 qtnf_debugfs_add_entry(bus, "mps", qtnf_dbg_mps_show);
1380 qtnf_debugfs_add_entry(bus, "msi_enabled", qtnf_dbg_msi_show);
1381 qtnf_debugfs_add_entry(bus, "hdp_stats", qtnf_dbg_hdp_stats);
1382 qtnf_debugfs_add_entry(bus, "irq_stats", qtnf_dbg_irq_stats);
1383 qtnf_debugfs_add_entry(bus, "shm_stats", qtnf_dbg_shm_stats);
1388 netif_napi_del(&bus->mux_napi);
1391 flush_workqueue(pcie_priv->workqueue);
1392 destroy_workqueue(pcie_priv->workqueue);
1395 pci_set_drvdata(pdev, NULL);
1401 static void qtnf_pcie_remove(struct pci_dev *pdev)
1403 struct qtnf_pcie_bus_priv *priv;
1404 struct qtnf_bus *bus;
1406 bus = pci_get_drvdata(pdev);
1410 priv = get_bus_priv(bus);
1412 qtnf_core_detach(bus);
1413 netif_napi_del(&bus->mux_napi);
1415 flush_workqueue(priv->workqueue);
1416 destroy_workqueue(priv->workqueue);
1417 tasklet_kill(&priv->reclaim_tq);
1419 qtnf_debugfs_remove(bus);
1421 qtnf_pcie_free_shm_ipc(priv);
1424 #ifdef CONFIG_PM_SLEEP
1425 static int qtnf_pcie_suspend(struct device *dev)
1430 static int qtnf_pcie_resume(struct device *dev)
1434 #endif /* CONFIG_PM_SLEEP */
1436 #ifdef CONFIG_PM_SLEEP
1437 /* Power Management Hooks */
1438 static SIMPLE_DEV_PM_OPS(qtnf_pcie_pm_ops, qtnf_pcie_suspend,
1442 static const struct pci_device_id qtnf_pcie_devid_table[] = {
1444 PCIE_VENDOR_ID_QUANTENNA, PCIE_DEVICE_ID_QTN_PEARL,
1445 PCI_ANY_ID, PCI_ANY_ID, 0, 0,
1450 MODULE_DEVICE_TABLE(pci, qtnf_pcie_devid_table);
1452 static struct pci_driver qtnf_pcie_drv_data = {
1454 .id_table = qtnf_pcie_devid_table,
1455 .probe = qtnf_pcie_probe,
1456 .remove = qtnf_pcie_remove,
1457 #ifdef CONFIG_PM_SLEEP
1459 .pm = &qtnf_pcie_pm_ops,
1464 static int __init qtnf_pcie_register(void)
1466 pr_info("register Quantenna QSR10g FullMAC PCIE driver\n");
1467 return pci_register_driver(&qtnf_pcie_drv_data);
1470 static void __exit qtnf_pcie_exit(void)
1472 pr_info("unregister Quantenna QSR10g FullMAC PCIE driver\n");
1473 pci_unregister_driver(&qtnf_pcie_drv_data);
1476 module_init(qtnf_pcie_register);
1477 module_exit(qtnf_pcie_exit);
1479 MODULE_AUTHOR("Quantenna Communications");
1480 MODULE_DESCRIPTION("Quantenna QSR10g PCIe bus driver for 802.11 wireless LAN.");
1481 MODULE_LICENSE("GPL");