1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
4 #include <linux/crash_dump.h>
5 #include <linux/etherdevice.h>
6 #include <linux/ethtool.h>
7 #include <linux/filter.h>
9 #include <linux/if_vlan.h>
10 #include <linux/module.h>
11 #include <linux/netdevice.h>
12 #include <linux/pci.h>
13 #include <linux/rtnetlink.h>
14 #include <linux/inetdevice.h>
17 #include "funeth_devlink.h"
18 #include "funeth_ktls.h"
20 #include "fun_queue.h"
21 #include "funeth_txrx.h"
23 #define ADMIN_SQ_DEPTH 32
24 #define ADMIN_CQ_DEPTH 64
25 #define ADMIN_RQ_DEPTH 16
27 /* Default number of Tx/Rx queues. */
28 #define FUN_DFLT_QUEUES 16U
31 FUN_SERV_RES_CHANGE = FUN_SERV_FIRST_AVAIL,
35 static const struct pci_device_id funeth_id_table[] = {
36 { PCI_VDEVICE(FUNGIBLE, 0x0101) },
37 { PCI_VDEVICE(FUNGIBLE, 0x0181) },
41 /* Issue a port write admin command with @n key/value pairs. */
42 static int fun_port_write_cmds(struct funeth_priv *fp, unsigned int n,
43 const int *keys, const u64 *data)
45 unsigned int cmd_size, i;
47 struct fun_admin_port_req req;
48 struct fun_admin_port_rsp rsp;
52 cmd_size = offsetof(struct fun_admin_port_req, u.write.write48) +
53 n * sizeof(struct fun_admin_write48_req);
54 if (cmd_size > sizeof(cmd) || cmd_size > ADMIN_RSP_MAX_LEN)
57 cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_PORT,
60 FUN_ADMIN_PORT_WRITE_REQ_INIT(FUN_ADMIN_SUBOP_WRITE, 0,
61 fp->netdev->dev_port);
62 for (i = 0; i < n; i++)
63 cmd.req.u.write.write48[i] =
64 FUN_ADMIN_WRITE48_REQ_INIT(keys[i], data[i]);
66 return fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common,
67 &cmd.rsp, cmd_size, 0);
70 int fun_port_write_cmd(struct funeth_priv *fp, int key, u64 data)
72 return fun_port_write_cmds(fp, 1, &key, &data);
75 /* Issue a port read admin command with @n key/value pairs. */
76 static int fun_port_read_cmds(struct funeth_priv *fp, unsigned int n,
77 const int *keys, u64 *data)
79 const struct fun_admin_read48_rsp *r48rsp;
80 unsigned int cmd_size, i;
83 struct fun_admin_port_req req;
84 struct fun_admin_port_rsp rsp;
88 cmd_size = offsetof(struct fun_admin_port_req, u.read.read48) +
89 n * sizeof(struct fun_admin_read48_req);
90 if (cmd_size > sizeof(cmd) || cmd_size > ADMIN_RSP_MAX_LEN)
93 cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_PORT,
96 FUN_ADMIN_PORT_READ_REQ_INIT(FUN_ADMIN_SUBOP_READ, 0,
97 fp->netdev->dev_port);
98 for (i = 0; i < n; i++)
99 cmd.req.u.read.read48[i] = FUN_ADMIN_READ48_REQ_INIT(keys[i]);
101 rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common,
102 &cmd.rsp, cmd_size, 0);
106 for (r48rsp = cmd.rsp.u.read.read48, i = 0; i < n; i++, r48rsp++) {
107 data[i] = FUN_ADMIN_READ48_RSP_DATA_G(r48rsp->key_to_data);
108 dev_dbg(fp->fdev->dev,
109 "port_read_rsp lport=%u (key_to_data=0x%llx) key=%d data:%lld retval:%lld",
110 fp->lport, r48rsp->key_to_data, keys[i], data[i],
111 FUN_ADMIN_READ48_RSP_RET_G(r48rsp->key_to_data));
116 int fun_port_read_cmd(struct funeth_priv *fp, int key, u64 *data)
118 return fun_port_read_cmds(fp, 1, &key, data);
121 static void fun_report_link(struct net_device *netdev)
123 if (netif_carrier_ok(netdev)) {
124 const struct funeth_priv *fp = netdev_priv(netdev);
125 const char *fec = "", *pause = "";
126 int speed = fp->link_speed;
129 if (fp->link_speed >= SPEED_1000) {
134 if (fp->active_fec & FUN_PORT_FEC_RS)
136 else if (fp->active_fec & FUN_PORT_FEC_FC)
139 if ((fp->active_fc & FUN_PORT_CAP_PAUSE_MASK) == FUN_PORT_CAP_PAUSE_MASK)
140 pause = ", Tx/Rx PAUSE";
141 else if (fp->active_fc & FUN_PORT_CAP_RX_PAUSE)
142 pause = ", Rx PAUSE";
143 else if (fp->active_fc & FUN_PORT_CAP_TX_PAUSE)
144 pause = ", Tx PAUSE";
146 netdev_info(netdev, "Link up at %d %cb/s full-duplex%s%s\n",
147 speed, unit, pause, fec);
149 netdev_info(netdev, "Link down\n");
153 static int fun_adi_write(struct fun_dev *fdev, enum fun_admin_adi_attr attr,
154 unsigned int adi_id, const struct fun_adi_param *param)
156 struct fun_admin_adi_req req = {
157 .common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_ADI,
159 .u.write.subop = FUN_ADMIN_SUBOP_WRITE,
160 .u.write.attribute = attr,
161 .u.write.id = cpu_to_be32(adi_id),
162 .u.write.param = *param
165 return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);
168 /* Configure RSS for the given port. @op determines whether a new RSS context
169 * is to be created or whether an existing one should be reconfigured. The
170 * remaining parameters specify the hashing algorithm, key, and indirection
173 * This initiates packet delivery to the Rx queues set in the indirection
176 int fun_config_rss(struct net_device *dev, int algo, const u8 *key,
177 const u32 *qtable, u8 op)
179 struct funeth_priv *fp = netdev_priv(dev);
180 unsigned int table_len = fp->indir_table_nentries;
181 unsigned int len = FUN_ETH_RSS_MAX_KEY_SIZE + sizeof(u32) * table_len;
182 struct funeth_rxq **rxqs = rtnl_dereference(fp->rxqs);
185 struct fun_admin_rss_req req;
186 struct fun_dataop_gl gl;
188 struct fun_admin_generic_create_rsp rsp;
194 if (op != FUN_ADMIN_SUBOP_CREATE && fp->rss_hw_id == FUN_HCI_ID_INVALID)
197 flags = op == FUN_ADMIN_SUBOP_CREATE ?
198 FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR : 0;
199 cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_RSS,
202 FUN_ADMIN_RSS_CREATE_REQ_INIT(op, flags, fp->rss_hw_id,
204 FUN_ETH_RSS_MAX_KEY_SIZE,
206 FUN_ETH_RSS_MAX_KEY_SIZE);
207 cmd.req.u.create.dataop = FUN_DATAOP_HDR_INIT(1, 0, 1, 0, len);
208 fun_dataop_gl_init(&cmd.gl, 0, 0, len, fp->rss_dma_addr);
210 /* write the key and indirection table into the RSS DMA area */
211 memcpy(fp->rss_cfg, key, FUN_ETH_RSS_MAX_KEY_SIZE);
212 indir_tab = fp->rss_cfg + FUN_ETH_RSS_MAX_KEY_SIZE;
213 for (rc = 0; rc < table_len; rc++)
214 *indir_tab++ = cpu_to_be32(rxqs[*qtable++]->hw_cqid);
216 rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common,
217 &cmd.rsp, sizeof(cmd.rsp), 0);
218 if (!rc && op == FUN_ADMIN_SUBOP_CREATE)
219 fp->rss_hw_id = be32_to_cpu(cmd.rsp.id);
223 /* Destroy the HW RSS conntext associated with the given port. This also stops
224 * all packet delivery to our Rx queues.
226 static void fun_destroy_rss(struct funeth_priv *fp)
228 if (fp->rss_hw_id != FUN_HCI_ID_INVALID) {
229 fun_res_destroy(fp->fdev, FUN_ADMIN_OP_RSS, 0, fp->rss_hw_id);
230 fp->rss_hw_id = FUN_HCI_ID_INVALID;
234 static void fun_irq_aff_notify(struct irq_affinity_notify *notify,
235 const cpumask_t *mask)
237 struct fun_irq *p = container_of(notify, struct fun_irq, aff_notify);
239 cpumask_copy(&p->affinity_mask, mask);
242 static void fun_irq_aff_release(struct kref __always_unused *ref)
246 /* Allocate an IRQ structure, assign an MSI-X index and initial affinity to it,
247 * and add it to the IRQ XArray.
249 static struct fun_irq *fun_alloc_qirq(struct funeth_priv *fp, unsigned int idx,
250 int node, unsigned int xa_idx_offset)
255 cpu = cpumask_local_spread(idx, node);
256 node = cpu_to_mem(cpu);
258 irq = kzalloc_node(sizeof(*irq), GFP_KERNEL, node);
260 return ERR_PTR(-ENOMEM);
262 res = fun_reserve_irqs(fp->fdev, 1, &irq->irq_idx);
266 res = xa_insert(&fp->irqs, idx + xa_idx_offset, irq, GFP_KERNEL);
270 irq->irq = pci_irq_vector(fp->pdev, irq->irq_idx);
271 cpumask_set_cpu(cpu, &irq->affinity_mask);
272 irq->aff_notify.notify = fun_irq_aff_notify;
273 irq->aff_notify.release = fun_irq_aff_release;
274 irq->state = FUN_IRQ_INIT;
278 fun_release_irqs(fp->fdev, 1, &irq->irq_idx);
284 static void fun_free_qirq(struct funeth_priv *fp, struct fun_irq *irq)
286 netif_napi_del(&irq->napi);
287 fun_release_irqs(fp->fdev, 1, &irq->irq_idx);
291 /* Release the IRQs reserved for Tx/Rx queues that aren't being used. */
292 static void fun_prune_queue_irqs(struct net_device *dev)
294 struct funeth_priv *fp = netdev_priv(dev);
295 unsigned int nreleased = 0;
299 xa_for_each(&fp->irqs, idx, irq) {
300 if (irq->txq || irq->rxq) /* skip those in use */
303 xa_erase(&fp->irqs, idx);
304 fun_free_qirq(fp, irq);
306 if (idx < fp->rx_irq_ofst)
311 netif_info(fp, intr, dev, "Released %u queue IRQs\n", nreleased);
314 /* Reserve IRQs, one per queue, to acommodate the requested queue numbers @ntx
315 * and @nrx. IRQs are added incrementally to those we already have.
316 * We hold on to allocated IRQs until garbage collection of unused IRQs is
317 * separately requested.
319 static int fun_alloc_queue_irqs(struct net_device *dev, unsigned int ntx,
322 struct funeth_priv *fp = netdev_priv(dev);
323 int node = dev_to_node(&fp->pdev->dev);
327 for (i = fp->num_tx_irqs; i < ntx; i++) {
328 irq = fun_alloc_qirq(fp, i, node, 0);
333 netif_napi_add_tx(dev, &irq->napi, fun_txq_napi_poll);
336 for (i = fp->num_rx_irqs; i < nrx; i++) {
337 irq = fun_alloc_qirq(fp, i, node, fp->rx_irq_ofst);
342 netif_napi_add(dev, &irq->napi, fun_rxq_napi_poll,
346 netif_info(fp, intr, dev, "Reserved %u/%u IRQs for Tx/Rx queues\n",
351 static void free_txqs(struct funeth_txq **txqs, unsigned int nqs,
352 unsigned int start, int state)
356 for (i = start; i < nqs && txqs[i]; i++)
357 txqs[i] = funeth_txq_free(txqs[i], state);
360 static int alloc_txqs(struct net_device *dev, struct funeth_txq **txqs,
361 unsigned int nqs, unsigned int depth, unsigned int start,
364 struct funeth_priv *fp = netdev_priv(dev);
368 for (i = start; i < nqs; i++) {
369 err = funeth_txq_create(dev, i, depth, xa_load(&fp->irqs, i),
372 free_txqs(txqs, nqs, start, FUN_QSTATE_DESTROYED);
379 static void free_rxqs(struct funeth_rxq **rxqs, unsigned int nqs,
380 unsigned int start, int state)
384 for (i = start; i < nqs && rxqs[i]; i++)
385 rxqs[i] = funeth_rxq_free(rxqs[i], state);
388 static int alloc_rxqs(struct net_device *dev, struct funeth_rxq **rxqs,
389 unsigned int nqs, unsigned int ncqe, unsigned int nrqe,
390 unsigned int start, int state)
392 struct funeth_priv *fp = netdev_priv(dev);
396 for (i = start; i < nqs; i++) {
397 err = funeth_rxq_create(dev, i, ncqe, nrqe,
398 xa_load(&fp->irqs, i + fp->rx_irq_ofst),
401 free_rxqs(rxqs, nqs, start, FUN_QSTATE_DESTROYED);
408 static void free_xdpqs(struct funeth_txq **xdpqs, unsigned int nqs,
409 unsigned int start, int state)
413 for (i = start; i < nqs && xdpqs[i]; i++)
414 xdpqs[i] = funeth_txq_free(xdpqs[i], state);
416 if (state == FUN_QSTATE_DESTROYED)
420 static struct funeth_txq **alloc_xdpqs(struct net_device *dev, unsigned int nqs,
421 unsigned int depth, unsigned int start,
424 struct funeth_txq **xdpqs;
428 xdpqs = kcalloc(nqs, sizeof(*xdpqs), GFP_KERNEL);
430 return ERR_PTR(-ENOMEM);
432 for (i = start; i < nqs; i++) {
433 err = funeth_txq_create(dev, i, depth, NULL, state, &xdpqs[i]);
435 free_xdpqs(xdpqs, nqs, start, FUN_QSTATE_DESTROYED);
442 static void fun_free_rings(struct net_device *netdev, struct fun_qset *qset)
444 struct funeth_priv *fp = netdev_priv(netdev);
445 struct funeth_txq **xdpqs = qset->xdpqs;
446 struct funeth_rxq **rxqs = qset->rxqs;
448 /* qset may not specify any queues to operate on. In that case the
449 * currently installed queues are implied.
452 rxqs = rtnl_dereference(fp->rxqs);
453 xdpqs = rtnl_dereference(fp->xdpqs);
454 qset->txqs = fp->txqs;
455 qset->nrxqs = netdev->real_num_rx_queues;
456 qset->ntxqs = netdev->real_num_tx_queues;
457 qset->nxdpqs = fp->num_xdpqs;
462 if (rxqs == rtnl_dereference(fp->rxqs)) {
463 rcu_assign_pointer(fp->rxqs, NULL);
464 rcu_assign_pointer(fp->xdpqs, NULL);
469 free_rxqs(rxqs, qset->nrxqs, qset->rxq_start, qset->state);
470 free_txqs(qset->txqs, qset->ntxqs, qset->txq_start, qset->state);
471 free_xdpqs(xdpqs, qset->nxdpqs, qset->xdpq_start, qset->state);
472 if (qset->state == FUN_QSTATE_DESTROYED)
475 /* Tell the caller which queues were operated on. */
480 static int fun_alloc_rings(struct net_device *netdev, struct fun_qset *qset)
482 struct funeth_txq **xdpqs = NULL, **txqs;
483 struct funeth_rxq **rxqs;
486 err = fun_alloc_queue_irqs(netdev, qset->ntxqs, qset->nrxqs);
490 rxqs = kcalloc(qset->ntxqs + qset->nrxqs, sizeof(*rxqs), GFP_KERNEL);
495 xdpqs = alloc_xdpqs(netdev, qset->nxdpqs, qset->sq_depth,
496 qset->xdpq_start, qset->state);
498 err = PTR_ERR(xdpqs);
503 txqs = (struct funeth_txq **)&rxqs[qset->nrxqs];
504 err = alloc_txqs(netdev, txqs, qset->ntxqs, qset->sq_depth,
505 qset->txq_start, qset->state);
509 err = alloc_rxqs(netdev, rxqs, qset->nrxqs, qset->cq_depth,
510 qset->rq_depth, qset->rxq_start, qset->state);
520 free_txqs(txqs, qset->ntxqs, qset->txq_start, FUN_QSTATE_DESTROYED);
522 free_xdpqs(xdpqs, qset->nxdpqs, qset->xdpq_start, FUN_QSTATE_DESTROYED);
528 /* Take queues to the next level. Presently this means creating them on the
531 static int fun_advance_ring_state(struct net_device *dev, struct fun_qset *qset)
533 struct funeth_priv *fp = netdev_priv(dev);
536 for (i = 0; i < qset->nrxqs; i++) {
537 err = fun_rxq_create_dev(qset->rxqs[i],
539 i + fp->rx_irq_ofst));
544 for (i = 0; i < qset->ntxqs; i++) {
545 err = fun_txq_create_dev(qset->txqs[i], xa_load(&fp->irqs, i));
550 for (i = 0; i < qset->nxdpqs; i++) {
551 err = fun_txq_create_dev(qset->xdpqs[i], NULL);
559 fun_free_rings(dev, qset);
563 static int fun_port_create(struct net_device *netdev)
565 struct funeth_priv *fp = netdev_priv(netdev);
567 struct fun_admin_port_req req;
568 struct fun_admin_port_rsp rsp;
572 if (fp->lport != INVALID_LPORT)
575 cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_PORT,
578 FUN_ADMIN_PORT_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, 0,
581 rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common, &cmd.rsp,
585 fp->lport = be16_to_cpu(cmd.rsp.u.create.lport);
589 static int fun_port_destroy(struct net_device *netdev)
591 struct funeth_priv *fp = netdev_priv(netdev);
593 if (fp->lport == INVALID_LPORT)
596 fp->lport = INVALID_LPORT;
597 return fun_res_destroy(fp->fdev, FUN_ADMIN_OP_PORT, 0,
601 static int fun_eth_create(struct funeth_priv *fp)
604 struct fun_admin_eth_req req;
605 struct fun_admin_generic_create_rsp rsp;
609 cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_ETH,
611 cmd.req.u.create = FUN_ADMIN_ETH_CREATE_REQ_INIT(
612 FUN_ADMIN_SUBOP_CREATE,
613 FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR,
614 0, fp->netdev->dev_port);
616 rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common, &cmd.rsp,
618 return rc ? rc : be32_to_cpu(cmd.rsp.id);
621 static int fun_vi_create(struct funeth_priv *fp)
623 struct fun_admin_vi_req req = {
624 .common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_VI,
626 .u.create = FUN_ADMIN_VI_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE,
628 fp->netdev->dev_port,
629 fp->netdev->dev_port)
632 return fun_submit_admin_sync_cmd(fp->fdev, &req.common, NULL, 0, 0);
635 /* Helper to create an ETH flow and bind an SQ to it.
636 * Returns the ETH id (>= 0) on success or a negative error.
638 int fun_create_and_bind_tx(struct funeth_priv *fp, u32 sqid)
642 ethid = fun_eth_create(fp);
644 rc = fun_bind(fp->fdev, FUN_ADMIN_BIND_TYPE_EPSQ, sqid,
645 FUN_ADMIN_BIND_TYPE_ETH, ethid);
647 fun_res_destroy(fp->fdev, FUN_ADMIN_OP_ETH, 0, ethid);
654 static irqreturn_t fun_queue_irq_handler(int irq, void *data)
656 struct fun_irq *p = data;
659 prefetch(p->rxq->next_cqe_info);
662 napi_schedule_irqoff(&p->napi);
666 static int fun_enable_irqs(struct net_device *dev)
668 struct funeth_priv *fp = netdev_priv(dev);
669 unsigned long idx, last;
675 xa_for_each(&fp->irqs, idx, p) {
686 if (p->state != FUN_IRQ_INIT)
689 snprintf(p->name, sizeof(p->name) - 1, "%s-%s-%u", dev->name,
691 err = request_irq(p->irq, fun_queue_irq_handler, 0, p->name, p);
693 netdev_err(dev, "Failed to allocate IRQ %u, err %d\n",
697 p->state = FUN_IRQ_REQUESTED;
700 xa_for_each(&fp->irqs, idx, p) {
701 if (p->state != FUN_IRQ_REQUESTED)
703 irq_set_affinity_notifier(p->irq, &p->aff_notify);
704 irq_set_affinity_and_hint(p->irq, &p->affinity_mask);
705 napi_enable(&p->napi);
706 p->state = FUN_IRQ_ENABLED;
713 xa_for_each_range(&fp->irqs, idx, p, 0, last)
714 if (p->state == FUN_IRQ_REQUESTED) {
716 p->state = FUN_IRQ_INIT;
722 static void fun_disable_one_irq(struct fun_irq *irq)
724 napi_disable(&irq->napi);
725 irq_set_affinity_notifier(irq->irq, NULL);
726 irq_update_affinity_hint(irq->irq, NULL);
727 free_irq(irq->irq, irq);
728 irq->state = FUN_IRQ_INIT;
731 static void fun_disable_irqs(struct net_device *dev)
733 struct funeth_priv *fp = netdev_priv(dev);
737 xa_for_each(&fp->irqs, idx, p)
738 if (p->state == FUN_IRQ_ENABLED)
739 fun_disable_one_irq(p);
742 static void fun_down(struct net_device *dev, struct fun_qset *qset)
744 struct funeth_priv *fp = netdev_priv(dev);
746 /* If we don't have queues the data path is already down.
747 * Note netif_running(dev) may be true.
749 if (!rcu_access_pointer(fp->rxqs))
752 /* It is also down if the queues aren't on the device. */
753 if (fp->txqs[0]->init_state >= FUN_QSTATE_INIT_FULL) {
754 netif_info(fp, ifdown, dev,
755 "Tearing down data path on device\n");
756 fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_DISABLE, 0);
758 netif_carrier_off(dev);
759 netif_tx_disable(dev);
762 fun_res_destroy(fp->fdev, FUN_ADMIN_OP_VI, 0, dev->dev_port);
763 fun_disable_irqs(dev);
766 fun_free_rings(dev, qset);
769 static int fun_up(struct net_device *dev, struct fun_qset *qset)
771 static const int port_keys[] = {
772 FUN_ADMIN_PORT_KEY_STATS_DMA_LOW,
773 FUN_ADMIN_PORT_KEY_STATS_DMA_HIGH,
774 FUN_ADMIN_PORT_KEY_ENABLE
777 struct funeth_priv *fp = netdev_priv(dev);
779 lower_32_bits(fp->stats_dma_addr),
780 upper_32_bits(fp->stats_dma_addr),
781 FUN_PORT_FLAG_ENABLE_NOTIFY
785 netif_info(fp, ifup, dev, "Setting up data path on device\n");
787 if (qset->rxqs[0]->init_state < FUN_QSTATE_INIT_FULL) {
788 err = fun_advance_ring_state(dev, qset);
793 err = fun_vi_create(fp);
797 fp->txqs = qset->txqs;
798 rcu_assign_pointer(fp->rxqs, qset->rxqs);
799 rcu_assign_pointer(fp->xdpqs, qset->xdpqs);
801 err = fun_enable_irqs(dev);
806 err = fun_config_rss(dev, fp->hash_algo, fp->rss_key,
807 fp->indir_table, FUN_ADMIN_SUBOP_CREATE);
809 /* The non-RSS case has only 1 queue. */
810 err = fun_bind(fp->fdev, FUN_ADMIN_BIND_TYPE_VI, dev->dev_port,
811 FUN_ADMIN_BIND_TYPE_EPCQ,
812 qset->rxqs[0]->hw_cqid);
817 err = fun_port_write_cmds(fp, 3, port_keys, vals);
821 netif_tx_start_all_queues(dev);
827 fun_disable_irqs(dev);
829 fun_res_destroy(fp->fdev, FUN_ADMIN_OP_VI, 0, dev->dev_port);
831 fun_free_rings(dev, qset);
835 static int funeth_open(struct net_device *netdev)
837 struct funeth_priv *fp = netdev_priv(netdev);
838 struct fun_qset qset = {
839 .nrxqs = netdev->real_num_rx_queues,
840 .ntxqs = netdev->real_num_tx_queues,
841 .nxdpqs = fp->num_xdpqs,
842 .cq_depth = fp->cq_depth,
843 .rq_depth = fp->rq_depth,
844 .sq_depth = fp->sq_depth,
845 .state = FUN_QSTATE_INIT_FULL,
849 rc = fun_alloc_rings(netdev, &qset);
853 rc = fun_up(netdev, &qset);
855 qset.state = FUN_QSTATE_DESTROYED;
856 fun_free_rings(netdev, &qset);
862 static int funeth_close(struct net_device *netdev)
864 struct fun_qset qset = { .state = FUN_QSTATE_DESTROYED };
866 fun_down(netdev, &qset);
870 static void fun_get_stats64(struct net_device *netdev,
871 struct rtnl_link_stats64 *stats)
873 struct funeth_priv *fp = netdev_priv(netdev);
874 struct funeth_txq **xdpqs;
875 struct funeth_rxq **rxqs;
876 unsigned int i, start;
878 stats->tx_packets = fp->tx_packets;
879 stats->tx_bytes = fp->tx_bytes;
880 stats->tx_dropped = fp->tx_dropped;
882 stats->rx_packets = fp->rx_packets;
883 stats->rx_bytes = fp->rx_bytes;
884 stats->rx_dropped = fp->rx_dropped;
887 rxqs = rcu_dereference(fp->rxqs);
891 for (i = 0; i < netdev->real_num_tx_queues; i++) {
892 struct funeth_txq_stats txs;
894 FUN_QSTAT_READ(fp->txqs[i], start, txs);
895 stats->tx_packets += txs.tx_pkts;
896 stats->tx_bytes += txs.tx_bytes;
897 stats->tx_dropped += txs.tx_map_err;
900 for (i = 0; i < netdev->real_num_rx_queues; i++) {
901 struct funeth_rxq_stats rxs;
903 FUN_QSTAT_READ(rxqs[i], start, rxs);
904 stats->rx_packets += rxs.rx_pkts;
905 stats->rx_bytes += rxs.rx_bytes;
906 stats->rx_dropped += rxs.rx_map_err + rxs.rx_mem_drops;
909 xdpqs = rcu_dereference(fp->xdpqs);
913 for (i = 0; i < fp->num_xdpqs; i++) {
914 struct funeth_txq_stats txs;
916 FUN_QSTAT_READ(xdpqs[i], start, txs);
917 stats->tx_packets += txs.tx_pkts;
918 stats->tx_bytes += txs.tx_bytes;
924 static int fun_change_mtu(struct net_device *netdev, int new_mtu)
926 struct funeth_priv *fp = netdev_priv(netdev);
929 rc = fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_MTU, new_mtu);
931 netdev->mtu = new_mtu;
935 static int fun_set_macaddr(struct net_device *netdev, void *addr)
937 struct funeth_priv *fp = netdev_priv(netdev);
938 struct sockaddr *saddr = addr;
941 if (!is_valid_ether_addr(saddr->sa_data))
942 return -EADDRNOTAVAIL;
944 if (ether_addr_equal(netdev->dev_addr, saddr->sa_data))
947 rc = fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_MACADDR,
948 ether_addr_to_u64(saddr->sa_data));
950 eth_hw_addr_set(netdev, saddr->sa_data);
954 static int fun_get_port_attributes(struct net_device *netdev)
956 static const int keys[] = {
957 FUN_ADMIN_PORT_KEY_MACADDR, FUN_ADMIN_PORT_KEY_CAPABILITIES,
958 FUN_ADMIN_PORT_KEY_ADVERT, FUN_ADMIN_PORT_KEY_MTU
960 static const int phys_keys[] = {
961 FUN_ADMIN_PORT_KEY_LANE_ATTRS,
964 struct funeth_priv *fp = netdev_priv(netdev);
965 u64 data[ARRAY_SIZE(keys)];
969 rc = fun_port_read_cmds(fp, ARRAY_SIZE(keys), keys, data);
973 for (i = 0; i < ARRAY_SIZE(keys); i++) {
975 case FUN_ADMIN_PORT_KEY_MACADDR:
976 u64_to_ether_addr(data[i], mac);
977 if (is_zero_ether_addr(mac)) {
978 eth_hw_addr_random(netdev);
979 } else if (is_valid_ether_addr(mac)) {
980 eth_hw_addr_set(netdev, mac);
983 "device provided a bad MAC address %pM\n",
989 case FUN_ADMIN_PORT_KEY_CAPABILITIES:
990 fp->port_caps = data[i];
993 case FUN_ADMIN_PORT_KEY_ADVERT:
994 fp->advertising = data[i];
997 case FUN_ADMIN_PORT_KEY_MTU:
998 netdev->mtu = data[i];
1003 if (!(fp->port_caps & FUN_PORT_CAP_VPORT)) {
1004 rc = fun_port_read_cmds(fp, ARRAY_SIZE(phys_keys), phys_keys,
1009 fp->lane_attrs = data[0];
1012 if (netdev->addr_assign_type == NET_ADDR_RANDOM)
1013 return fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_MACADDR,
1014 ether_addr_to_u64(netdev->dev_addr));
1018 static int fun_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
1020 const struct funeth_priv *fp = netdev_priv(dev);
1022 return copy_to_user(ifr->ifr_data, &fp->hwtstamp_cfg,
1023 sizeof(fp->hwtstamp_cfg)) ? -EFAULT : 0;
1026 static int fun_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
1028 struct funeth_priv *fp = netdev_priv(dev);
1029 struct hwtstamp_config cfg;
1031 if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1034 /* no TX HW timestamps */
1035 cfg.tx_type = HWTSTAMP_TX_OFF;
1037 switch (cfg.rx_filter) {
1038 case HWTSTAMP_FILTER_NONE:
1040 case HWTSTAMP_FILTER_ALL:
1041 case HWTSTAMP_FILTER_SOME:
1042 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1043 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1044 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1045 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1046 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1047 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1048 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1049 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1050 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1051 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1052 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1053 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1054 case HWTSTAMP_FILTER_NTP_ALL:
1055 cfg.rx_filter = HWTSTAMP_FILTER_ALL;
1061 fp->hwtstamp_cfg = cfg;
1062 return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1065 static int fun_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
1069 return fun_hwtstamp_set(dev, ifr);
1071 return fun_hwtstamp_get(dev, ifr);
1077 /* Prepare the queues for XDP. */
1078 static int fun_enter_xdp(struct net_device *dev, struct bpf_prog *prog)
1080 struct funeth_priv *fp = netdev_priv(dev);
1081 unsigned int i, nqs = num_online_cpus();
1082 struct funeth_txq **xdpqs;
1083 struct funeth_rxq **rxqs;
1086 xdpqs = alloc_xdpqs(dev, nqs, fp->sq_depth, 0, FUN_QSTATE_INIT_FULL);
1088 return PTR_ERR(xdpqs);
1090 rxqs = rtnl_dereference(fp->rxqs);
1091 for (i = 0; i < dev->real_num_rx_queues; i++) {
1092 err = fun_rxq_set_bpf(rxqs[i], prog);
1097 fp->num_xdpqs = nqs;
1098 rcu_assign_pointer(fp->xdpqs, xdpqs);
1102 fun_rxq_set_bpf(rxqs[i], NULL);
1104 free_xdpqs(xdpqs, nqs, 0, FUN_QSTATE_DESTROYED);
1108 /* Set the queues for non-XDP operation. */
1109 static void fun_end_xdp(struct net_device *dev)
1111 struct funeth_priv *fp = netdev_priv(dev);
1112 struct funeth_txq **xdpqs;
1113 struct funeth_rxq **rxqs;
1116 xdpqs = rtnl_dereference(fp->xdpqs);
1117 rcu_assign_pointer(fp->xdpqs, NULL);
1119 /* at this point both Rx and Tx XDP processing has ended */
1121 free_xdpqs(xdpqs, fp->num_xdpqs, 0, FUN_QSTATE_DESTROYED);
1124 rxqs = rtnl_dereference(fp->rxqs);
1125 for (i = 0; i < dev->real_num_rx_queues; i++)
1126 fun_rxq_set_bpf(rxqs[i], NULL);
1129 #define XDP_MAX_MTU \
1130 (PAGE_SIZE - FUN_XDP_HEADROOM - VLAN_ETH_HLEN - FUN_RX_TAILROOM)
1132 static int fun_xdp_setup(struct net_device *dev, struct netdev_bpf *xdp)
1134 struct bpf_prog *old_prog, *prog = xdp->prog;
1135 struct funeth_priv *fp = netdev_priv(dev);
1138 /* XDP uses at most one buffer */
1139 if (prog && dev->mtu > XDP_MAX_MTU) {
1140 netdev_err(dev, "device MTU %u too large for XDP\n", dev->mtu);
1141 NL_SET_ERR_MSG_MOD(xdp->extack,
1142 "Device MTU too large for XDP");
1146 if (!netif_running(dev)) {
1147 fp->num_xdpqs = prog ? num_online_cpus() : 0;
1148 } else if (prog && !fp->xdp_prog) {
1149 err = fun_enter_xdp(dev, prog);
1151 NL_SET_ERR_MSG_MOD(xdp->extack,
1152 "Failed to set queues for XDP.");
1155 } else if (!prog && fp->xdp_prog) {
1158 struct funeth_rxq **rxqs = rtnl_dereference(fp->rxqs);
1160 for (i = 0; i < dev->real_num_rx_queues; i++)
1161 WRITE_ONCE(rxqs[i]->xdp_prog, prog);
1164 dev->max_mtu = prog ? XDP_MAX_MTU : FUN_MAX_MTU;
1165 old_prog = xchg(&fp->xdp_prog, prog);
1167 bpf_prog_put(old_prog);
1172 static int fun_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1174 switch (xdp->command) {
1175 case XDP_SETUP_PROG:
1176 return fun_xdp_setup(dev, xdp);
1182 static struct devlink_port *fun_get_devlink_port(struct net_device *netdev)
1184 struct funeth_priv *fp = netdev_priv(netdev);
1186 return &fp->dl_port;
1189 static int fun_init_vports(struct fun_ethdev *ed, unsigned int n)
1194 ed->vport_info = kvcalloc(n, sizeof(*ed->vport_info), GFP_KERNEL);
1195 if (!ed->vport_info)
1201 static void fun_free_vports(struct fun_ethdev *ed)
1203 kvfree(ed->vport_info);
1204 ed->vport_info = NULL;
1208 static struct fun_vport_info *fun_get_vport(struct fun_ethdev *ed,
1211 if (!ed->vport_info || vport >= ed->num_vports)
1214 return ed->vport_info + vport;
1217 static int fun_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
1219 struct funeth_priv *fp = netdev_priv(dev);
1220 struct fun_adi_param mac_param = {};
1221 struct fun_dev *fdev = fp->fdev;
1222 struct fun_ethdev *ed = to_fun_ethdev(fdev);
1223 struct fun_vport_info *vi;
1226 if (is_multicast_ether_addr(mac))
1229 mutex_lock(&ed->state_mutex);
1230 vi = fun_get_vport(ed, vf);
1234 mac_param.u.mac = FUN_ADI_MAC_INIT(ether_addr_to_u64(mac));
1235 rc = fun_adi_write(fdev, FUN_ADMIN_ADI_ATTR_MACADDR, vf + 1,
1238 ether_addr_copy(vi->mac, mac);
1240 mutex_unlock(&ed->state_mutex);
1244 static int fun_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
1247 struct funeth_priv *fp = netdev_priv(dev);
1248 struct fun_adi_param vlan_param = {};
1249 struct fun_dev *fdev = fp->fdev;
1250 struct fun_ethdev *ed = to_fun_ethdev(fdev);
1251 struct fun_vport_info *vi;
1254 if (vlan > 4095 || qos > 7)
1256 if (vlan_proto && vlan_proto != htons(ETH_P_8021Q) &&
1257 vlan_proto != htons(ETH_P_8021AD))
1260 mutex_lock(&ed->state_mutex);
1261 vi = fun_get_vport(ed, vf);
1265 vlan_param.u.vlan = FUN_ADI_VLAN_INIT(be16_to_cpu(vlan_proto),
1266 ((u16)qos << VLAN_PRIO_SHIFT) | vlan);
1267 rc = fun_adi_write(fdev, FUN_ADMIN_ADI_ATTR_VLAN, vf + 1, &vlan_param);
1271 vi->vlan_proto = vlan_proto;
1274 mutex_unlock(&ed->state_mutex);
1278 static int fun_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
1281 struct funeth_priv *fp = netdev_priv(dev);
1282 struct fun_adi_param rate_param = {};
1283 struct fun_dev *fdev = fp->fdev;
1284 struct fun_ethdev *ed = to_fun_ethdev(fdev);
1285 struct fun_vport_info *vi;
1291 mutex_lock(&ed->state_mutex);
1292 vi = fun_get_vport(ed, vf);
1296 rate_param.u.rate = FUN_ADI_RATE_INIT(max_tx_rate);
1297 rc = fun_adi_write(fdev, FUN_ADMIN_ADI_ATTR_RATE, vf + 1, &rate_param);
1299 vi->max_rate = max_tx_rate;
1301 mutex_unlock(&ed->state_mutex);
1305 static int fun_get_vf_config(struct net_device *dev, int vf,
1306 struct ifla_vf_info *ivi)
1308 struct funeth_priv *fp = netdev_priv(dev);
1309 struct fun_ethdev *ed = to_fun_ethdev(fp->fdev);
1310 const struct fun_vport_info *vi;
1312 mutex_lock(&ed->state_mutex);
1313 vi = fun_get_vport(ed, vf);
1317 memset(ivi, 0, sizeof(*ivi));
1319 ether_addr_copy(ivi->mac, vi->mac);
1320 ivi->vlan = vi->vlan;
1322 ivi->vlan_proto = vi->vlan_proto;
1323 ivi->max_tx_rate = vi->max_rate;
1324 ivi->spoofchk = vi->spoofchk;
1326 mutex_unlock(&ed->state_mutex);
1327 return vi ? 0 : -EINVAL;
1330 static void fun_uninit(struct net_device *dev)
1332 struct funeth_priv *fp = netdev_priv(dev);
1334 fun_prune_queue_irqs(dev);
1335 xa_destroy(&fp->irqs);
1338 static const struct net_device_ops fun_netdev_ops = {
1339 .ndo_open = funeth_open,
1340 .ndo_stop = funeth_close,
1341 .ndo_start_xmit = fun_start_xmit,
1342 .ndo_get_stats64 = fun_get_stats64,
1343 .ndo_change_mtu = fun_change_mtu,
1344 .ndo_set_mac_address = fun_set_macaddr,
1345 .ndo_validate_addr = eth_validate_addr,
1346 .ndo_eth_ioctl = fun_ioctl,
1347 .ndo_uninit = fun_uninit,
1349 .ndo_xdp_xmit = fun_xdp_xmit_frames,
1350 .ndo_set_vf_mac = fun_set_vf_mac,
1351 .ndo_set_vf_vlan = fun_set_vf_vlan,
1352 .ndo_set_vf_rate = fun_set_vf_rate,
1353 .ndo_get_vf_config = fun_get_vf_config,
1354 .ndo_get_devlink_port = fun_get_devlink_port,
1357 #define GSO_ENCAP_FLAGS (NETIF_F_GSO_GRE | NETIF_F_GSO_IPXIP4 | \
1358 NETIF_F_GSO_IPXIP6 | NETIF_F_GSO_UDP_TUNNEL | \
1359 NETIF_F_GSO_UDP_TUNNEL_CSUM)
1360 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
1361 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_HW_CSUM | TSO_FLAGS | \
1362 GSO_ENCAP_FLAGS | NETIF_F_HIGHDMA)
1364 static void fun_dflt_rss_indir(struct funeth_priv *fp, unsigned int nrx)
1368 for (i = 0; i < fp->indir_table_nentries; i++)
1369 fp->indir_table[i] = ethtool_rxfh_indir_default(i, nrx);
1372 /* Reset the RSS indirection table to equal distribution across the current
1373 * number of Rx queues. Called at init time and whenever the number of Rx
1374 * queues changes subsequently. Note that this may also resize the indirection
1377 static void fun_reset_rss_indir(struct net_device *dev, unsigned int nrx)
1379 struct funeth_priv *fp = netdev_priv(dev);
1384 /* Set the table size to the max possible that allows an equal number
1385 * of occurrences of each CQ.
1387 fp->indir_table_nentries = rounddown(FUN_ETH_RSS_MAX_INDIR_ENT, nrx);
1388 fun_dflt_rss_indir(fp, nrx);
1391 /* Update the RSS LUT to contain only queues in [0, nrx). Normally this will
1392 * update the LUT to an equal distribution among nrx queues, If @only_if_needed
1393 * is set the LUT is left unchanged if it already does not reference any queues
1396 static int fun_rss_set_qnum(struct net_device *dev, unsigned int nrx,
1397 bool only_if_needed)
1399 struct funeth_priv *fp = netdev_priv(dev);
1400 u32 old_lut[FUN_ETH_RSS_MAX_INDIR_ENT];
1401 unsigned int i, oldsz;
1407 if (only_if_needed) {
1408 for (i = 0; i < fp->indir_table_nentries; i++)
1409 if (fp->indir_table[i] >= nrx)
1412 if (i >= fp->indir_table_nentries)
1416 memcpy(old_lut, fp->indir_table, sizeof(old_lut));
1417 oldsz = fp->indir_table_nentries;
1418 fun_reset_rss_indir(dev, nrx);
1420 err = fun_config_rss(dev, fp->hash_algo, fp->rss_key,
1421 fp->indir_table, FUN_ADMIN_SUBOP_MODIFY);
1425 memcpy(fp->indir_table, old_lut, sizeof(old_lut));
1426 fp->indir_table_nentries = oldsz;
1430 /* Allocate the DMA area for the RSS configuration commands to the device, and
1431 * initialize the hash, hash key, indirection table size and its entries to
1432 * their defaults. The indirection table defaults to equal distribution across
1435 static int fun_init_rss(struct net_device *dev)
1437 struct funeth_priv *fp = netdev_priv(dev);
1438 size_t size = sizeof(fp->rss_key) + sizeof(fp->indir_table);
1440 fp->rss_hw_id = FUN_HCI_ID_INVALID;
1441 if (!(fp->port_caps & FUN_PORT_CAP_OFFLOADS))
1444 fp->rss_cfg = dma_alloc_coherent(&fp->pdev->dev, size,
1445 &fp->rss_dma_addr, GFP_KERNEL);
1449 fp->hash_algo = FUN_ETH_RSS_ALG_TOEPLITZ;
1450 netdev_rss_key_fill(fp->rss_key, sizeof(fp->rss_key));
1451 fun_reset_rss_indir(dev, dev->real_num_rx_queues);
1455 static void fun_free_rss(struct funeth_priv *fp)
1458 dma_free_coherent(&fp->pdev->dev,
1459 sizeof(fp->rss_key) + sizeof(fp->indir_table),
1460 fp->rss_cfg, fp->rss_dma_addr);
1465 void fun_set_ring_count(struct net_device *netdev, unsigned int ntx,
1468 netif_set_real_num_tx_queues(netdev, ntx);
1469 if (nrx != netdev->real_num_rx_queues) {
1470 netif_set_real_num_rx_queues(netdev, nrx);
1471 fun_reset_rss_indir(netdev, nrx);
1475 static int fun_init_stats_area(struct funeth_priv *fp)
1477 unsigned int nstats;
1479 if (!(fp->port_caps & FUN_PORT_CAP_STATS))
1482 nstats = PORT_MAC_RX_STATS_MAX + PORT_MAC_TX_STATS_MAX +
1483 PORT_MAC_FEC_STATS_MAX;
1485 fp->stats = dma_alloc_coherent(&fp->pdev->dev, nstats * sizeof(u64),
1486 &fp->stats_dma_addr, GFP_KERNEL);
1492 static void fun_free_stats_area(struct funeth_priv *fp)
1494 unsigned int nstats;
1497 nstats = PORT_MAC_RX_STATS_MAX + PORT_MAC_TX_STATS_MAX;
1498 dma_free_coherent(&fp->pdev->dev, nstats * sizeof(u64),
1499 fp->stats, fp->stats_dma_addr);
1504 static int fun_dl_port_register(struct net_device *netdev)
1506 struct funeth_priv *fp = netdev_priv(netdev);
1507 struct devlink *dl = priv_to_devlink(fp->fdev);
1508 struct devlink_port_attrs attrs = {};
1511 if (fp->port_caps & FUN_PORT_CAP_VPORT) {
1512 attrs.flavour = DEVLINK_PORT_FLAVOUR_VIRTUAL;
1515 idx = netdev->dev_port;
1516 attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
1517 attrs.lanes = fp->lane_attrs & 7;
1518 if (fp->lane_attrs & FUN_PORT_LANE_SPLIT) {
1520 attrs.phys.port_number = fp->lport & ~3;
1521 attrs.phys.split_subport_number = fp->lport & 3;
1523 attrs.phys.port_number = fp->lport;
1527 devlink_port_attrs_set(&fp->dl_port, &attrs);
1529 return devlink_port_register(dl, &fp->dl_port, idx);
1532 /* Determine the max Tx/Rx queues for a port. */
1533 static int fun_max_qs(struct fun_ethdev *ed, unsigned int *ntx,
1538 if (ed->num_ports > 1 || is_kdump_kernel()) {
1544 neth = fun_get_res_count(&ed->fdev, FUN_ADMIN_OP_ETH);
1548 /* We determine the max number of queues based on the CPU
1549 * cores, device interrupts and queues, RSS size, and device Tx flows.
1551 * - At least 1 Rx and 1 Tx queues.
1552 * - At most 1 Rx/Tx queue per core.
1553 * - Each Rx/Tx queue needs 1 SQ.
1555 *ntx = min(ed->nsqs_per_port - 1, num_online_cpus());
1559 if (*nrx > FUN_ETH_RSS_MAX_INDIR_ENT)
1560 *nrx = FUN_ETH_RSS_MAX_INDIR_ENT;
1564 static void fun_queue_defaults(struct net_device *dev, unsigned int nsqs)
1566 unsigned int ntx, nrx;
1568 ntx = min(dev->num_tx_queues, FUN_DFLT_QUEUES);
1569 nrx = min(dev->num_rx_queues, FUN_DFLT_QUEUES);
1571 ntx = min(ntx, nsqs / 2);
1572 nrx = min(nrx, nsqs - ntx);
1574 nrx = min(nrx, nsqs / 2);
1575 ntx = min(ntx, nsqs - nrx);
1578 netif_set_real_num_tx_queues(dev, ntx);
1579 netif_set_real_num_rx_queues(dev, nrx);
1582 /* Replace the existing Rx/Tx/XDP queues with equal number of queues with
1583 * different settings, e.g. depth. This is a disruptive replacement that
1584 * temporarily shuts down the data path and should be limited to changes that
1585 * can't be applied to live queues. The old queues are always discarded.
1587 int fun_replace_queues(struct net_device *dev, struct fun_qset *newqs,
1588 struct netlink_ext_ack *extack)
1590 struct fun_qset oldqs = { .state = FUN_QSTATE_DESTROYED };
1591 struct funeth_priv *fp = netdev_priv(dev);
1594 newqs->nrxqs = dev->real_num_rx_queues;
1595 newqs->ntxqs = dev->real_num_tx_queues;
1596 newqs->nxdpqs = fp->num_xdpqs;
1597 newqs->state = FUN_QSTATE_INIT_SW;
1598 err = fun_alloc_rings(dev, newqs);
1600 NL_SET_ERR_MSG_MOD(extack,
1601 "Unable to allocate memory for new queues, keeping current settings");
1605 fun_down(dev, &oldqs);
1607 err = fun_up(dev, newqs);
1611 /* The new queues couldn't be installed. We do not retry the old queues
1612 * as they are the same to the device as the new queues and would
1615 newqs->state = FUN_QSTATE_DESTROYED;
1616 fun_free_rings(dev, newqs);
1617 NL_SET_ERR_MSG_MOD(extack, "Unable to restore the data path with the new queues.");
1621 /* Change the number of Rx/Tx queues of a device while it is up. This is done
1622 * by incrementally adding/removing queues to meet the new requirements while
1623 * handling ongoing traffic.
1625 int fun_change_num_queues(struct net_device *dev, unsigned int ntx,
1628 unsigned int keep_tx = min(dev->real_num_tx_queues, ntx);
1629 unsigned int keep_rx = min(dev->real_num_rx_queues, nrx);
1630 struct funeth_priv *fp = netdev_priv(dev);
1631 struct fun_qset oldqs = {
1632 .rxqs = rtnl_dereference(fp->rxqs),
1634 .nrxqs = dev->real_num_rx_queues,
1635 .ntxqs = dev->real_num_tx_queues,
1636 .rxq_start = keep_rx,
1637 .txq_start = keep_tx,
1638 .state = FUN_QSTATE_DESTROYED
1640 struct fun_qset newqs = {
1643 .rxq_start = keep_rx,
1644 .txq_start = keep_tx,
1645 .cq_depth = fp->cq_depth,
1646 .rq_depth = fp->rq_depth,
1647 .sq_depth = fp->sq_depth,
1648 .state = FUN_QSTATE_INIT_FULL
1652 err = fun_alloc_rings(dev, &newqs);
1656 err = fun_enable_irqs(dev); /* of any newly added queues */
1660 /* copy the queues we are keeping to the new set */
1661 memcpy(newqs.rxqs, oldqs.rxqs, keep_rx * sizeof(*oldqs.rxqs));
1662 memcpy(newqs.txqs, fp->txqs, keep_tx * sizeof(*fp->txqs));
1664 if (nrx < dev->real_num_rx_queues) {
1665 err = fun_rss_set_qnum(dev, nrx, true);
1667 goto disable_tx_irqs;
1669 for (i = nrx; i < dev->real_num_rx_queues; i++)
1670 fun_disable_one_irq(container_of(oldqs.rxqs[i]->napi,
1671 struct fun_irq, napi));
1673 netif_set_real_num_rx_queues(dev, nrx);
1676 if (ntx < dev->real_num_tx_queues)
1677 netif_set_real_num_tx_queues(dev, ntx);
1679 rcu_assign_pointer(fp->rxqs, newqs.rxqs);
1680 fp->txqs = newqs.txqs;
1683 if (ntx > dev->real_num_tx_queues)
1684 netif_set_real_num_tx_queues(dev, ntx);
1686 if (nrx > dev->real_num_rx_queues) {
1687 netif_set_real_num_rx_queues(dev, nrx);
1688 fun_rss_set_qnum(dev, nrx, false);
1691 /* disable interrupts of any excess Tx queues */
1692 for (i = keep_tx; i < oldqs.ntxqs; i++)
1693 fun_disable_one_irq(oldqs.txqs[i]->irq);
1695 fun_free_rings(dev, &oldqs);
1696 fun_prune_queue_irqs(dev);
1700 for (i = oldqs.ntxqs; i < ntx; i++)
1701 fun_disable_one_irq(newqs.txqs[i]->irq);
1703 newqs.state = FUN_QSTATE_DESTROYED;
1704 fun_free_rings(dev, &newqs);
1706 fun_prune_queue_irqs(dev);
1710 static int fun_create_netdev(struct fun_ethdev *ed, unsigned int portid)
1712 struct fun_dev *fdev = &ed->fdev;
1713 struct net_device *netdev;
1714 struct funeth_priv *fp;
1715 unsigned int ntx, nrx;
1718 rc = fun_max_qs(ed, &ntx, &nrx);
1722 netdev = alloc_etherdev_mqs(sizeof(*fp), ntx, nrx);
1728 netdev->dev_port = portid;
1729 fun_queue_defaults(netdev, ed->nsqs_per_port);
1731 fp = netdev_priv(netdev);
1733 fp->pdev = to_pci_dev(fdev->dev);
1734 fp->netdev = netdev;
1736 fp->rx_irq_ofst = ntx;
1737 seqcount_init(&fp->link_seq);
1739 fp->lport = INVALID_LPORT;
1740 rc = fun_port_create(netdev);
1744 /* bind port to admin CQ for async events */
1745 rc = fun_bind(fdev, FUN_ADMIN_BIND_TYPE_PORT, portid,
1746 FUN_ADMIN_BIND_TYPE_EPCQ, 0);
1750 rc = fun_get_port_attributes(netdev);
1754 rc = fun_init_rss(netdev);
1758 rc = fun_init_stats_area(fp);
1762 SET_NETDEV_DEV(netdev, fdev->dev);
1763 netdev->netdev_ops = &fun_netdev_ops;
1765 netdev->hw_features = NETIF_F_SG | NETIF_F_RXHASH | NETIF_F_RXCSUM;
1766 if (fp->port_caps & FUN_PORT_CAP_OFFLOADS)
1767 netdev->hw_features |= NETIF_F_HW_CSUM | TSO_FLAGS;
1768 if (fp->port_caps & FUN_PORT_CAP_ENCAP_OFFLOADS)
1769 netdev->hw_features |= GSO_ENCAP_FLAGS;
1771 netdev->features |= netdev->hw_features | NETIF_F_HIGHDMA;
1772 netdev->vlan_features = netdev->features & VLAN_FEAT;
1773 netdev->mpls_features = netdev->vlan_features;
1774 netdev->hw_enc_features = netdev->hw_features;
1776 netdev->min_mtu = ETH_MIN_MTU;
1777 netdev->max_mtu = FUN_MAX_MTU;
1779 fun_set_ethtool_ops(netdev);
1781 /* configurable parameters */
1782 fp->sq_depth = min(SQ_DEPTH, fdev->q_depth);
1783 fp->cq_depth = min(CQ_DEPTH, fdev->q_depth);
1784 fp->rq_depth = min_t(unsigned int, RQ_DEPTH, fdev->q_depth);
1785 fp->rx_coal_usec = CQ_INTCOAL_USEC;
1786 fp->rx_coal_count = CQ_INTCOAL_NPKT;
1787 fp->tx_coal_usec = SQ_INTCOAL_USEC;
1788 fp->tx_coal_count = SQ_INTCOAL_NPKT;
1789 fp->cq_irq_db = FUN_IRQ_CQ_DB(fp->rx_coal_usec, fp->rx_coal_count);
1791 rc = fun_dl_port_register(netdev);
1795 fp->ktls_id = FUN_HCI_ID_INVALID;
1796 fun_ktls_init(netdev); /* optional, failure OK */
1798 netif_carrier_off(netdev);
1799 ed->netdevs[portid] = netdev;
1800 rc = register_netdev(netdev);
1804 if (fp->dl_port.devlink)
1805 devlink_port_type_eth_set(&fp->dl_port, netdev);
1810 ed->netdevs[portid] = NULL;
1811 fun_ktls_cleanup(fp);
1812 if (fp->dl_port.devlink)
1813 devlink_port_unregister(&fp->dl_port);
1815 fun_free_stats_area(fp);
1819 fun_port_destroy(netdev);
1821 free_netdev(netdev);
1823 dev_err(fdev->dev, "couldn't allocate port %u, error %d", portid, rc);
1827 static void fun_destroy_netdev(struct net_device *netdev)
1829 struct funeth_priv *fp;
1831 fp = netdev_priv(netdev);
1832 if (fp->dl_port.devlink) {
1833 devlink_port_type_clear(&fp->dl_port);
1834 devlink_port_unregister(&fp->dl_port);
1836 unregister_netdev(netdev);
1837 fun_ktls_cleanup(fp);
1838 fun_free_stats_area(fp);
1840 fun_port_destroy(netdev);
1841 free_netdev(netdev);
1844 static int fun_create_ports(struct fun_ethdev *ed, unsigned int nports)
1846 struct fun_dev *fd = &ed->fdev;
1849 /* The admin queue takes 1 IRQ and 2 SQs. */
1850 ed->nsqs_per_port = min(fd->num_irqs - 1,
1851 fd->kern_end_qid - 2) / nports;
1852 if (ed->nsqs_per_port < 2) {
1853 dev_err(fd->dev, "Too few SQs for %u ports", nports);
1857 ed->netdevs = kcalloc(nports, sizeof(*ed->netdevs), GFP_KERNEL);
1861 ed->num_ports = nports;
1862 for (i = 0; i < nports; i++) {
1863 rc = fun_create_netdev(ed, i);
1872 fun_destroy_netdev(ed->netdevs[--i]);
1879 static void fun_destroy_ports(struct fun_ethdev *ed)
1883 for (i = 0; i < ed->num_ports; i++)
1884 fun_destroy_netdev(ed->netdevs[i]);
1891 static void fun_update_link_state(const struct fun_ethdev *ed,
1892 const struct fun_admin_port_notif *notif)
1894 unsigned int port_idx = be16_to_cpu(notif->id);
1895 struct net_device *netdev;
1896 struct funeth_priv *fp;
1898 if (port_idx >= ed->num_ports)
1901 netdev = ed->netdevs[port_idx];
1902 fp = netdev_priv(netdev);
1904 write_seqcount_begin(&fp->link_seq);
1905 fp->link_speed = be32_to_cpu(notif->speed) * 10; /* 10 Mbps->Mbps */
1906 fp->active_fc = notif->flow_ctrl;
1907 fp->active_fec = notif->fec;
1908 fp->xcvr_type = notif->xcvr_type;
1909 fp->link_down_reason = notif->link_down_reason;
1910 fp->lp_advertising = be64_to_cpu(notif->lp_advertising);
1912 if ((notif->link_state | notif->missed_events) & FUN_PORT_FLAG_MAC_DOWN)
1913 netif_carrier_off(netdev);
1914 if (notif->link_state & FUN_PORT_FLAG_MAC_UP)
1915 netif_carrier_on(netdev);
1917 write_seqcount_end(&fp->link_seq);
1918 fun_report_link(netdev);
1921 /* handler for async events delivered through the admin CQ */
1922 static void fun_event_cb(struct fun_dev *fdev, void *entry)
1924 u8 op = ((struct fun_admin_rsp_common *)entry)->op;
1926 if (op == FUN_ADMIN_OP_PORT) {
1927 const struct fun_admin_port_notif *rsp = entry;
1929 if (rsp->subop == FUN_ADMIN_SUBOP_NOTIFY) {
1930 fun_update_link_state(to_fun_ethdev(fdev), rsp);
1931 } else if (rsp->subop == FUN_ADMIN_SUBOP_RES_COUNT) {
1932 const struct fun_admin_res_count_rsp *r = entry;
1935 set_bit(FUN_SERV_RES_CHANGE, &fdev->service_flags);
1937 set_bit(FUN_SERV_DEL_PORTS, &fdev->service_flags);
1938 fun_serv_sched(fdev);
1940 dev_info(fdev->dev, "adminq event unexpected op %u subop %u",
1944 dev_info(fdev->dev, "adminq event unexpected op %u", op);
1948 /* handler for pending work managed by the service task */
1949 static void fun_service_cb(struct fun_dev *fdev)
1951 struct fun_ethdev *ed = to_fun_ethdev(fdev);
1954 if (test_and_clear_bit(FUN_SERV_DEL_PORTS, &fdev->service_flags))
1955 fun_destroy_ports(ed);
1957 if (!test_and_clear_bit(FUN_SERV_RES_CHANGE, &fdev->service_flags))
1960 rc = fun_get_res_count(fdev, FUN_ADMIN_OP_PORT);
1961 if (rc < 0 || rc == ed->num_ports)
1965 fun_destroy_ports(ed);
1967 fun_create_ports(ed, rc);
1970 static int funeth_sriov_configure(struct pci_dev *pdev, int nvfs)
1972 struct fun_dev *fdev = pci_get_drvdata(pdev);
1973 struct fun_ethdev *ed = to_fun_ethdev(fdev);
1977 if (pci_vfs_assigned(pdev)) {
1978 dev_warn(&pdev->dev,
1979 "Cannot disable SR-IOV while VFs are assigned\n");
1983 mutex_lock(&ed->state_mutex);
1984 fun_free_vports(ed);
1985 mutex_unlock(&ed->state_mutex);
1986 pci_disable_sriov(pdev);
1990 rc = pci_enable_sriov(pdev, nvfs);
1994 mutex_lock(&ed->state_mutex);
1995 rc = fun_init_vports(ed, nvfs);
1996 mutex_unlock(&ed->state_mutex);
1998 pci_disable_sriov(pdev);
2005 static int funeth_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2007 struct fun_dev_params aqreq = {
2008 .cqe_size_log2 = ilog2(ADMIN_CQE_SIZE),
2009 .sqe_size_log2 = ilog2(ADMIN_SQE_SIZE),
2010 .cq_depth = ADMIN_CQ_DEPTH,
2011 .sq_depth = ADMIN_SQ_DEPTH,
2012 .rq_depth = ADMIN_RQ_DEPTH,
2013 .min_msix = 2, /* 1 Rx + 1 Tx */
2014 .event_cb = fun_event_cb,
2015 .serv_cb = fun_service_cb,
2017 struct devlink *devlink;
2018 struct fun_ethdev *ed;
2019 struct fun_dev *fdev;
2022 devlink = fun_devlink_alloc(&pdev->dev);
2024 dev_err(&pdev->dev, "devlink alloc failed\n");
2028 ed = devlink_priv(devlink);
2029 mutex_init(&ed->state_mutex);
2032 rc = fun_dev_enable(fdev, pdev, &aqreq, KBUILD_MODNAME);
2036 rc = fun_get_res_count(fdev, FUN_ADMIN_OP_PORT);
2038 rc = fun_create_ports(ed, rc);
2042 fun_serv_restart(fdev);
2043 fun_devlink_register(devlink);
2047 fun_dev_disable(fdev);
2049 mutex_destroy(&ed->state_mutex);
2050 fun_devlink_free(devlink);
2054 static void funeth_remove(struct pci_dev *pdev)
2056 struct fun_dev *fdev = pci_get_drvdata(pdev);
2057 struct devlink *devlink;
2058 struct fun_ethdev *ed;
2060 ed = to_fun_ethdev(fdev);
2061 devlink = priv_to_devlink(ed);
2062 fun_devlink_unregister(devlink);
2064 #ifdef CONFIG_PCI_IOV
2065 funeth_sriov_configure(pdev, 0);
2068 fun_serv_stop(fdev);
2069 fun_destroy_ports(ed);
2070 fun_dev_disable(fdev);
2071 mutex_destroy(&ed->state_mutex);
2073 fun_devlink_free(devlink);
2076 static struct pci_driver funeth_driver = {
2077 .name = KBUILD_MODNAME,
2078 .id_table = funeth_id_table,
2079 .probe = funeth_probe,
2080 .remove = funeth_remove,
2081 .shutdown = funeth_remove,
2082 .sriov_configure = funeth_sriov_configure,
2085 module_pci_driver(funeth_driver);
2087 MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");
2088 MODULE_DESCRIPTION("Fungible Ethernet Network Driver");
2089 MODULE_LICENSE("Dual BSD/GPL");
2090 MODULE_DEVICE_TABLE(pci, funeth_id_table);