drivers/net/ethernet/fungible/funeth/funeth_main.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2
   3 #include <linux/bpf.h>
   4 #include <linux/crash_dump.h>
   5 #include <linux/etherdevice.h>
   6 #include <linux/ethtool.h>
   7 #include <linux/filter.h>
   8 #include <linux/idr.h>
   9 #include <linux/if_vlan.h>
  10 #include <linux/module.h>
  11 #include <linux/netdevice.h>
  12 #include <linux/pci.h>
  13 #include <linux/rtnetlink.h>
  14 #include <linux/inetdevice.h>
  15
  16 #include "funeth.h"
  17 #include "funeth_devlink.h"
  18 #include "funeth_ktls.h"
  19 #include "fun_port.h"
  20 #include "fun_queue.h"
  21 #include "funeth_txrx.h"
  22
  23 #define ADMIN_SQ_DEPTH 32
  24 #define ADMIN_CQ_DEPTH 64
  25 #define ADMIN_RQ_DEPTH 16
  26
  27 /* Default number of Tx/Rx queues. */
  28 #define FUN_DFLT_QUEUES 16U
  29
  30 enum {
  31         FUN_SERV_RES_CHANGE = FUN_SERV_FIRST_AVAIL,
  32         FUN_SERV_DEL_PORTS,
  33 };
  34
  35 static const struct pci_device_id funeth_id_table[] = {
  36         { PCI_VDEVICE(FUNGIBLE, 0x0101) },
  37         { PCI_VDEVICE(FUNGIBLE, 0x0181) },
  38         { 0, }
  39 };
  40
  41 /* Issue a port write admin command with @n key/value pairs. */
  42 static int fun_port_write_cmds(struct funeth_priv *fp, unsigned int n,
  43                                const int *keys, const u64 *data)
  44 {
  45         unsigned int cmd_size, i;
  46         union {
  47                 struct fun_admin_port_req req;
  48                 struct fun_admin_port_rsp rsp;
  49                 u8 v[ADMIN_SQE_SIZE];
  50         } cmd;
  51
  52         cmd_size = offsetof(struct fun_admin_port_req, u.write.write48) +
  53                 n * sizeof(struct fun_admin_write48_req);
  54         if (cmd_size > sizeof(cmd) || cmd_size > ADMIN_RSP_MAX_LEN)
  55                 return -EINVAL;
  56
  57         cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_PORT,
  58                                                     cmd_size);
  59         cmd.req.u.write =
  60                 FUN_ADMIN_PORT_WRITE_REQ_INIT(FUN_ADMIN_SUBOP_WRITE, 0,
  61                                               fp->netdev->dev_port);
  62         for (i = 0; i < n; i++)
  63                 cmd.req.u.write.write48[i] =
  64                         FUN_ADMIN_WRITE48_REQ_INIT(keys[i], data[i]);
  65
  66         return fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common,
  67                                          &cmd.rsp, cmd_size, 0);
  68 }
  69
  70 int fun_port_write_cmd(struct funeth_priv *fp, int key, u64 data)
  71 {
  72         return fun_port_write_cmds(fp, 1, &key, &data);
  73 }
  74
  75 /* Issue a port read admin command with @n key/value pairs. */
  76 static int fun_port_read_cmds(struct funeth_priv *fp, unsigned int n,
  77                               const int *keys, u64 *data)
  78 {
  79         const struct fun_admin_read48_rsp *r48rsp;
  80         unsigned int cmd_size, i;
  81         int rc;
  82         union {
  83                 struct fun_admin_port_req req;
  84                 struct fun_admin_port_rsp rsp;
  85                 u8 v[ADMIN_SQE_SIZE];
  86         } cmd;
  87
  88         cmd_size = offsetof(struct fun_admin_port_req, u.read.read48) +
  89                 n * sizeof(struct fun_admin_read48_req);
  90         if (cmd_size > sizeof(cmd) || cmd_size > ADMIN_RSP_MAX_LEN)
  91                 return -EINVAL;
  92
  93         cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_PORT,
  94                                                     cmd_size);
  95         cmd.req.u.read =
  96                 FUN_ADMIN_PORT_READ_REQ_INIT(FUN_ADMIN_SUBOP_READ, 0,
  97                                              fp->netdev->dev_port);
  98         for (i = 0; i < n; i++)
  99                 cmd.req.u.read.read48[i] = FUN_ADMIN_READ48_REQ_INIT(keys[i]);
 100
 101         rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common,
 102                                        &cmd.rsp, cmd_size, 0);
 103         if (rc)
 104                 return rc;
 105
 106         for (r48rsp = cmd.rsp.u.read.read48, i = 0; i < n; i++, r48rsp++) {
 107                 data[i] = FUN_ADMIN_READ48_RSP_DATA_G(r48rsp->key_to_data);
 108                 dev_dbg(fp->fdev->dev,
 109                         "port_read_rsp lport=%u (key_to_data=0x%llx) key=%d data:%lld retval:%lld",
 110                         fp->lport, r48rsp->key_to_data, keys[i], data[i],
 111                         FUN_ADMIN_READ48_RSP_RET_G(r48rsp->key_to_data));
 112         }
 113         return 0;
 114 }
 115
 116 int fun_port_read_cmd(struct funeth_priv *fp, int key, u64 *data)
 117 {
 118         return fun_port_read_cmds(fp, 1, &key, data);
 119 }
 120
 121 static void fun_report_link(struct net_device *netdev)
 122 {
 123         if (netif_carrier_ok(netdev)) {
 124                 const struct funeth_priv *fp = netdev_priv(netdev);
 125                 const char *fec = "", *pause = "";
 126                 int speed = fp->link_speed;
 127                 char unit = 'M';
 128
 129                 if (fp->link_speed >= SPEED_1000) {
 130                         speed /= 1000;
 131                         unit = 'G';
 132                 }
 133
 134                 if (fp->active_fec & FUN_PORT_FEC_RS)
 135                         fec = ", RS-FEC";
 136                 else if (fp->active_fec & FUN_PORT_FEC_FC)
 137                         fec = ", BASER-FEC";
 138
 139                 if ((fp->active_fc & FUN_PORT_CAP_PAUSE_MASK) == FUN_PORT_CAP_PAUSE_MASK)
 140                         pause = ", Tx/Rx PAUSE";
 141                 else if (fp->active_fc & FUN_PORT_CAP_RX_PAUSE)
 142                         pause = ", Rx PAUSE";
 143                 else if (fp->active_fc & FUN_PORT_CAP_TX_PAUSE)
 144                         pause = ", Tx PAUSE";
 145
 146                 netdev_info(netdev, "Link up at %d %cb/s full-duplex%s%s\n",
 147                             speed, unit, pause, fec);
 148         } else {
 149                 netdev_info(netdev, "Link down\n");
 150         }
 151 }
 152
 153 static int fun_adi_write(struct fun_dev *fdev, enum fun_admin_adi_attr attr,
 154                          unsigned int adi_id, const struct fun_adi_param *param)
 155 {
 156         struct fun_admin_adi_req req = {
 157                 .common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_ADI,
 158                                                      sizeof(req)),
 159                 .u.write.subop = FUN_ADMIN_SUBOP_WRITE,
 160                 .u.write.attribute = attr,
 161                 .u.write.id = cpu_to_be32(adi_id),
 162                 .u.write.param = *param
 163         };
 164
 165         return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);
 166 }
 167
 168 /* Configure RSS for the given port. @op determines whether a new RSS context
 169  * is to be created or whether an existing one should be reconfigured. The
 170  * remaining parameters specify the hashing algorithm, key, and indirection
 171  * table.
 172  *
 173  * This initiates packet delivery to the Rx queues set in the indirection
 174  * table.
 175  */
 176 int fun_config_rss(struct net_device *dev, int algo, const u8 *key,
 177                    const u32 *qtable, u8 op)
 178 {
 179         struct funeth_priv *fp = netdev_priv(dev);
 180         unsigned int table_len = fp->indir_table_nentries;
 181         unsigned int len = FUN_ETH_RSS_MAX_KEY_SIZE + sizeof(u32) * table_len;
 182         struct funeth_rxq **rxqs = rtnl_dereference(fp->rxqs);
 183         union {
 184                 struct {
 185                         struct fun_admin_rss_req req;
 186                         struct fun_dataop_gl gl;
 187                 };
 188                 struct fun_admin_generic_create_rsp rsp;
 189         } cmd;
 190         __be32 *indir_tab;
 191         u16 flags;
 192         int rc;
 193
 194         if (op != FUN_ADMIN_SUBOP_CREATE && fp->rss_hw_id == FUN_HCI_ID_INVALID)
 195                 return -EINVAL;
 196
 197         flags = op == FUN_ADMIN_SUBOP_CREATE ?
 198                         FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR : 0;
 199         cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_RSS,
 200                                                     sizeof(cmd));
 201         cmd.req.u.create =
 202                 FUN_ADMIN_RSS_CREATE_REQ_INIT(op, flags, fp->rss_hw_id,
 203                                               dev->dev_port, algo,
 204                                               FUN_ETH_RSS_MAX_KEY_SIZE,
 205                                               table_len, 0,
 206                                               FUN_ETH_RSS_MAX_KEY_SIZE);
 207         cmd.req.u.create.dataop = FUN_DATAOP_HDR_INIT(1, 0, 1, 0, len);
 208         fun_dataop_gl_init(&cmd.gl, 0, 0, len, fp->rss_dma_addr);
 209
 210         /* write the key and indirection table into the RSS DMA area */
 211         memcpy(fp->rss_cfg, key, FUN_ETH_RSS_MAX_KEY_SIZE);
 212         indir_tab = fp->rss_cfg + FUN_ETH_RSS_MAX_KEY_SIZE;
 213         for (rc = 0; rc < table_len; rc++)
 214                 *indir_tab++ = cpu_to_be32(rxqs[*qtable++]->hw_cqid);
 215
 216         rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common,
 217                                        &cmd.rsp, sizeof(cmd.rsp), 0);
 218         if (!rc && op == FUN_ADMIN_SUBOP_CREATE)
 219                 fp->rss_hw_id = be32_to_cpu(cmd.rsp.id);
 220         return rc;
 221 }
 222
 223 /* Destroy the HW RSS conntext associated with the given port. This also stops
 224  * all packet delivery to our Rx queues.
 225  */
 226 static void fun_destroy_rss(struct funeth_priv *fp)
 227 {
 228         if (fp->rss_hw_id != FUN_HCI_ID_INVALID) {
 229                 fun_res_destroy(fp->fdev, FUN_ADMIN_OP_RSS, 0, fp->rss_hw_id);
 230                 fp->rss_hw_id = FUN_HCI_ID_INVALID;
 231         }
 232 }
 233
 234 static void fun_irq_aff_notify(struct irq_affinity_notify *notify,
 235                                const cpumask_t *mask)
 236 {
 237         struct fun_irq *p = container_of(notify, struct fun_irq, aff_notify);
 238
 239         cpumask_copy(&p->affinity_mask, mask);
 240 }
 241
 242 static void fun_irq_aff_release(struct kref __always_unused *ref)
 243 {
 244 }
 245
 246 /* Allocate an IRQ structure, assign an MSI-X index and initial affinity to it,
 247  * and add it to the IRQ XArray.
 248  */
 249 static struct fun_irq *fun_alloc_qirq(struct funeth_priv *fp, unsigned int idx,
 250                                       int node, unsigned int xa_idx_offset)
 251 {
 252         struct fun_irq *irq;
 253         int cpu, res;
 254
 255         cpu = cpumask_local_spread(idx, node);
 256         node = cpu_to_mem(cpu);
 257
 258         irq = kzalloc_node(sizeof(*irq), GFP_KERNEL, node);
 259         if (!irq)
 260                 return ERR_PTR(-ENOMEM);
 261
 262         res = fun_reserve_irqs(fp->fdev, 1, &irq->irq_idx);
 263         if (res != 1)
 264                 goto free_irq;
 265
 266         res = xa_insert(&fp->irqs, idx + xa_idx_offset, irq, GFP_KERNEL);
 267         if (res)
 268                 goto release_irq;
 269
 270         irq->irq = pci_irq_vector(fp->pdev, irq->irq_idx);
 271         cpumask_set_cpu(cpu, &irq->affinity_mask);
 272         irq->aff_notify.notify = fun_irq_aff_notify;
 273         irq->aff_notify.release = fun_irq_aff_release;
 274         irq->state = FUN_IRQ_INIT;
 275         return irq;
 276
 277 release_irq:
 278         fun_release_irqs(fp->fdev, 1, &irq->irq_idx);
 279 free_irq:
 280         kfree(irq);
 281         return ERR_PTR(res);
 282 }
 283
 284 static void fun_free_qirq(struct funeth_priv *fp, struct fun_irq *irq)
 285 {
 286         netif_napi_del(&irq->napi);
 287         fun_release_irqs(fp->fdev, 1, &irq->irq_idx);
 288         kfree(irq);
 289 }
 290
 291 /* Release the IRQs reserved for Tx/Rx queues that aren't being used. */
 292 static void fun_prune_queue_irqs(struct net_device *dev)
 293 {
 294         struct funeth_priv *fp = netdev_priv(dev);
 295         unsigned int nreleased = 0;
 296         struct fun_irq *irq;
 297         unsigned long idx;
 298
 299         xa_for_each(&fp->irqs, idx, irq) {
 300                 if (irq->txq || irq->rxq)  /* skip those in use */
 301                         continue;
 302
 303                 xa_erase(&fp->irqs, idx);
 304                 fun_free_qirq(fp, irq);
 305                 nreleased++;
 306                 if (idx < fp->rx_irq_ofst)
 307                         fp->num_tx_irqs--;
 308                 else
 309                         fp->num_rx_irqs--;
 310         }
 311         netif_info(fp, intr, dev, "Released %u queue IRQs\n", nreleased);
 312 }
 313
 314 /* Reserve IRQs, one per queue, to acommodate the requested queue numbers @ntx
 315  * and @nrx. IRQs are added incrementally to those we already have.
 316  * We hold on to allocated IRQs until garbage collection of unused IRQs is
 317  * separately requested.
 318  */
 319 static int fun_alloc_queue_irqs(struct net_device *dev, unsigned int ntx,
 320                                 unsigned int nrx)
 321 {
 322         struct funeth_priv *fp = netdev_priv(dev);
 323         int node = dev_to_node(&fp->pdev->dev);
 324         struct fun_irq *irq;
 325         unsigned int i;
 326
 327         for (i = fp->num_tx_irqs; i < ntx; i++) {
 328                 irq = fun_alloc_qirq(fp, i, node, 0);
 329                 if (IS_ERR(irq))
 330                         return PTR_ERR(irq);
 331
 332                 fp->num_tx_irqs++;
 333                 netif_napi_add_tx(dev, &irq->napi, fun_txq_napi_poll);
 334         }
 335
 336         for (i = fp->num_rx_irqs; i < nrx; i++) {
 337                 irq = fun_alloc_qirq(fp, i, node, fp->rx_irq_ofst);
 338                 if (IS_ERR(irq))
 339                         return PTR_ERR(irq);
 340
 341                 fp->num_rx_irqs++;
 342                 netif_napi_add(dev, &irq->napi, fun_rxq_napi_poll,
 343                                NAPI_POLL_WEIGHT);
 344         }
 345
 346         netif_info(fp, intr, dev, "Reserved %u/%u IRQs for Tx/Rx queues\n",
 347                    ntx, nrx);
 348         return 0;
 349 }
 350
 351 static void free_txqs(struct funeth_txq **txqs, unsigned int nqs,
 352                       unsigned int start, int state)
 353 {
 354         unsigned int i;
 355
 356         for (i = start; i < nqs && txqs[i]; i++)
 357                 txqs[i] = funeth_txq_free(txqs[i], state);
 358 }
 359
 360 static int alloc_txqs(struct net_device *dev, struct funeth_txq **txqs,
 361                       unsigned int nqs, unsigned int depth, unsigned int start,
 362                       int state)
 363 {
 364         struct funeth_priv *fp = netdev_priv(dev);
 365         unsigned int i;
 366         int err;
 367
 368         for (i = start; i < nqs; i++) {
 369                 err = funeth_txq_create(dev, i, depth, xa_load(&fp->irqs, i),
 370                                         state, &txqs[i]);
 371                 if (err) {
 372                         free_txqs(txqs, nqs, start, FUN_QSTATE_DESTROYED);
 373                         return err;
 374                 }
 375         }
 376         return 0;
 377 }
 378
 379 static void free_rxqs(struct funeth_rxq **rxqs, unsigned int nqs,
 380                       unsigned int start, int state)
 381 {
 382         unsigned int i;
 383
 384         for (i = start; i < nqs && rxqs[i]; i++)
 385                 rxqs[i] = funeth_rxq_free(rxqs[i], state);
 386 }
 387
 388 static int alloc_rxqs(struct net_device *dev, struct funeth_rxq **rxqs,
 389                       unsigned int nqs, unsigned int ncqe, unsigned int nrqe,
 390                       unsigned int start, int state)
 391 {
 392         struct funeth_priv *fp = netdev_priv(dev);
 393         unsigned int i;
 394         int err;
 395
 396         for (i = start; i < nqs; i++) {
 397                 err = funeth_rxq_create(dev, i, ncqe, nrqe,
 398                                         xa_load(&fp->irqs, i + fp->rx_irq_ofst),
 399                                         state, &rxqs[i]);
 400                 if (err) {
 401                         free_rxqs(rxqs, nqs, start, FUN_QSTATE_DESTROYED);
 402                         return err;
 403                 }
 404         }
 405         return 0;
 406 }
 407
 408 static void free_xdpqs(struct funeth_txq **xdpqs, unsigned int nqs,
 409                        unsigned int start, int state)
 410 {
 411         unsigned int i;
 412
 413         for (i = start; i < nqs && xdpqs[i]; i++)
 414                 xdpqs[i] = funeth_txq_free(xdpqs[i], state);
 415
 416         if (state == FUN_QSTATE_DESTROYED)
 417                 kfree(xdpqs);
 418 }
 419
 420 static struct funeth_txq **alloc_xdpqs(struct net_device *dev, unsigned int nqs,
 421                                        unsigned int depth, unsigned int start,
 422                                        int state)
 423 {
 424         struct funeth_txq **xdpqs;
 425         unsigned int i;
 426         int err;
 427
 428         xdpqs = kcalloc(nqs, sizeof(*xdpqs), GFP_KERNEL);
 429         if (!xdpqs)
 430                 return ERR_PTR(-ENOMEM);
 431
 432         for (i = start; i < nqs; i++) {
 433                 err = funeth_txq_create(dev, i, depth, NULL, state, &xdpqs[i]);
 434                 if (err) {
 435                         free_xdpqs(xdpqs, nqs, start, FUN_QSTATE_DESTROYED);
 436                         return ERR_PTR(err);
 437                 }
 438         }
 439         return xdpqs;
 440 }
 441
 442 static void fun_free_rings(struct net_device *netdev, struct fun_qset *qset)
 443 {
 444         struct funeth_priv *fp = netdev_priv(netdev);
 445         struct funeth_txq **xdpqs = qset->xdpqs;
 446         struct funeth_rxq **rxqs = qset->rxqs;
 447
 448         /* qset may not specify any queues to operate on. In that case the
 449          * currently installed queues are implied.
 450          */
 451         if (!rxqs) {
 452                 rxqs = rtnl_dereference(fp->rxqs);
 453                 xdpqs = rtnl_dereference(fp->xdpqs);
 454                 qset->txqs = fp->txqs;
 455                 qset->nrxqs = netdev->real_num_rx_queues;
 456                 qset->ntxqs = netdev->real_num_tx_queues;
 457                 qset->nxdpqs = fp->num_xdpqs;
 458         }
 459         if (!rxqs)
 460                 return;
 461
 462         if (rxqs == rtnl_dereference(fp->rxqs)) {
 463                 rcu_assign_pointer(fp->rxqs, NULL);
 464                 rcu_assign_pointer(fp->xdpqs, NULL);
 465                 synchronize_net();
 466                 fp->txqs = NULL;
 467         }
 468
 469         free_rxqs(rxqs, qset->nrxqs, qset->rxq_start, qset->state);
 470         free_txqs(qset->txqs, qset->ntxqs, qset->txq_start, qset->state);
 471         free_xdpqs(xdpqs, qset->nxdpqs, qset->xdpq_start, qset->state);
 472         if (qset->state == FUN_QSTATE_DESTROYED)
 473                 kfree(rxqs);
 474
 475         /* Tell the caller which queues were operated on. */
 476         qset->rxqs = rxqs;
 477         qset->xdpqs = xdpqs;
 478 }
 479
 480 static int fun_alloc_rings(struct net_device *netdev, struct fun_qset *qset)
 481 {
 482         struct funeth_txq **xdpqs = NULL, **txqs;
 483         struct funeth_rxq **rxqs;
 484         int err;
 485
 486         err = fun_alloc_queue_irqs(netdev, qset->ntxqs, qset->nrxqs);
 487         if (err)
 488                 return err;
 489
 490         rxqs = kcalloc(qset->ntxqs + qset->nrxqs, sizeof(*rxqs), GFP_KERNEL);
 491         if (!rxqs)
 492                 return -ENOMEM;
 493
 494         if (qset->nxdpqs) {
 495                 xdpqs = alloc_xdpqs(netdev, qset->nxdpqs, qset->sq_depth,
 496                                     qset->xdpq_start, qset->state);
 497                 if (IS_ERR(xdpqs)) {
 498                         err = PTR_ERR(xdpqs);
 499                         goto free_qvec;
 500                 }
 501         }
 502
 503         txqs = (struct funeth_txq **)&rxqs[qset->nrxqs];
 504         err = alloc_txqs(netdev, txqs, qset->ntxqs, qset->sq_depth,
 505                          qset->txq_start, qset->state);
 506         if (err)
 507                 goto free_xdpqs;
 508
 509         err = alloc_rxqs(netdev, rxqs, qset->nrxqs, qset->cq_depth,
 510                          qset->rq_depth, qset->rxq_start, qset->state);
 511         if (err)
 512                 goto free_txqs;
 513
 514         qset->rxqs = rxqs;
 515         qset->txqs = txqs;
 516         qset->xdpqs = xdpqs;
 517         return 0;
 518
 519 free_txqs:
 520         free_txqs(txqs, qset->ntxqs, qset->txq_start, FUN_QSTATE_DESTROYED);
 521 free_xdpqs:
 522         free_xdpqs(xdpqs, qset->nxdpqs, qset->xdpq_start, FUN_QSTATE_DESTROYED);
 523 free_qvec:
 524         kfree(rxqs);
 525         return err;
 526 }
 527
 528 /* Take queues to the next level. Presently this means creating them on the
 529  * device.
 530  */
 531 static int fun_advance_ring_state(struct net_device *dev, struct fun_qset *qset)
 532 {
 533         struct funeth_priv *fp = netdev_priv(dev);
 534         int i, err;
 535
 536         for (i = 0; i < qset->nrxqs; i++) {
 537                 err = fun_rxq_create_dev(qset->rxqs[i],
 538                                          xa_load(&fp->irqs,
 539                                                  i + fp->rx_irq_ofst));
 540                 if (err)
 541                         goto out;
 542         }
 543
 544         for (i = 0; i < qset->ntxqs; i++) {
 545                 err = fun_txq_create_dev(qset->txqs[i], xa_load(&fp->irqs, i));
 546                 if (err)
 547                         goto out;
 548         }
 549
 550         for (i = 0; i < qset->nxdpqs; i++) {
 551                 err = fun_txq_create_dev(qset->xdpqs[i], NULL);
 552                 if (err)
 553                         goto out;
 554         }
 555
 556         return 0;
 557
 558 out:
 559         fun_free_rings(dev, qset);
 560         return err;
 561 }
 562
 563 static int fun_port_create(struct net_device *netdev)
 564 {
 565         struct funeth_priv *fp = netdev_priv(netdev);
 566         union {
 567                 struct fun_admin_port_req req;
 568                 struct fun_admin_port_rsp rsp;
 569         } cmd;
 570         int rc;
 571
 572         if (fp->lport != INVALID_LPORT)
 573                 return 0;
 574
 575         cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_PORT,
 576                                                     sizeof(cmd.req));
 577         cmd.req.u.create =
 578                 FUN_ADMIN_PORT_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, 0,
 579                                                netdev->dev_port);
 580
 581         rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common, &cmd.rsp,
 582                                        sizeof(cmd.rsp), 0);
 583
 584         if (!rc)
 585                 fp->lport = be16_to_cpu(cmd.rsp.u.create.lport);
 586         return rc;
 587 }
 588
 589 static int fun_port_destroy(struct net_device *netdev)
 590 {
 591         struct funeth_priv *fp = netdev_priv(netdev);
 592
 593         if (fp->lport == INVALID_LPORT)
 594                 return 0;
 595
 596         fp->lport = INVALID_LPORT;
 597         return fun_res_destroy(fp->fdev, FUN_ADMIN_OP_PORT, 0,
 598                                netdev->dev_port);
 599 }
 600
 601 static int fun_eth_create(struct funeth_priv *fp)
 602 {
 603         union {
 604                 struct fun_admin_eth_req req;
 605                 struct fun_admin_generic_create_rsp rsp;
 606         } cmd;
 607         int rc;
 608
 609         cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_ETH,
 610                                                     sizeof(cmd.req));
 611         cmd.req.u.create = FUN_ADMIN_ETH_CREATE_REQ_INIT(
 612                                 FUN_ADMIN_SUBOP_CREATE,
 613                                 FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR,
 614                                 0, fp->netdev->dev_port);
 615
 616         rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common, &cmd.rsp,
 617                                        sizeof(cmd.rsp), 0);
 618         return rc ? rc : be32_to_cpu(cmd.rsp.id);
 619 }
 620
 621 static int fun_vi_create(struct funeth_priv *fp)
 622 {
 623         struct fun_admin_vi_req req = {
 624                 .common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_VI,
 625                                                      sizeof(req)),
 626                 .u.create = FUN_ADMIN_VI_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE,
 627                                                          0,
 628                                                          fp->netdev->dev_port,
 629                                                          fp->netdev->dev_port)
 630         };
 631
 632         return fun_submit_admin_sync_cmd(fp->fdev, &req.common, NULL, 0, 0);
 633 }
 634
 635 /* Helper to create an ETH flow and bind an SQ to it.
 636  * Returns the ETH id (>= 0) on success or a negative error.
 637  */
 638 int fun_create_and_bind_tx(struct funeth_priv *fp, u32 sqid)
 639 {
 640         int rc, ethid;
 641
 642         ethid = fun_eth_create(fp);
 643         if (ethid >= 0) {
 644                 rc = fun_bind(fp->fdev, FUN_ADMIN_BIND_TYPE_EPSQ, sqid,
 645                               FUN_ADMIN_BIND_TYPE_ETH, ethid);
 646                 if (rc) {
 647                         fun_res_destroy(fp->fdev, FUN_ADMIN_OP_ETH, 0, ethid);
 648                         ethid = rc;
 649                 }
 650         }
 651         return ethid;
 652 }
 653
 654 static irqreturn_t fun_queue_irq_handler(int irq, void *data)
 655 {
 656         struct fun_irq *p = data;
 657
 658         if (p->rxq) {
 659                 prefetch(p->rxq->next_cqe_info);
 660                 p->rxq->irq_cnt++;
 661         }
 662         napi_schedule_irqoff(&p->napi);
 663         return IRQ_HANDLED;
 664 }
 665
 666 static int fun_enable_irqs(struct net_device *dev)
 667 {
 668         struct funeth_priv *fp = netdev_priv(dev);
 669         unsigned long idx, last;
 670         unsigned int qidx;
 671         struct fun_irq *p;
 672         const char *qtype;
 673         int err;
 674
 675         xa_for_each(&fp->irqs, idx, p) {
 676                 if (p->txq) {
 677                         qtype = "tx";
 678                         qidx = p->txq->qidx;
 679                 } else if (p->rxq) {
 680                         qtype = "rx";
 681                         qidx = p->rxq->qidx;
 682                 } else {
 683                         continue;
 684                 }
 685
 686                 if (p->state != FUN_IRQ_INIT)
 687                         continue;
 688
 689                 snprintf(p->name, sizeof(p->name) - 1, "%s-%s-%u", dev->name,
 690                          qtype, qidx);
 691                 err = request_irq(p->irq, fun_queue_irq_handler, 0, p->name, p);
 692                 if (err) {
 693                         netdev_err(dev, "Failed to allocate IRQ %u, err %d\n",
 694                                    p->irq, err);
 695                         goto unroll;
 696                 }
 697                 p->state = FUN_IRQ_REQUESTED;
 698         }
 699
 700         xa_for_each(&fp->irqs, idx, p) {
 701                 if (p->state != FUN_IRQ_REQUESTED)
 702                         continue;
 703                 irq_set_affinity_notifier(p->irq, &p->aff_notify);
 704                 irq_set_affinity_and_hint(p->irq, &p->affinity_mask);
 705                 napi_enable(&p->napi);
 706                 p->state = FUN_IRQ_ENABLED;
 707         }
 708
 709         return 0;
 710
 711 unroll:
 712         last = idx - 1;
 713         xa_for_each_range(&fp->irqs, idx, p, 0, last)
 714                 if (p->state == FUN_IRQ_REQUESTED) {
 715                         free_irq(p->irq, p);
 716                         p->state = FUN_IRQ_INIT;
 717                 }
 718
 719         return err;
 720 }
 721
 722 static void fun_disable_one_irq(struct fun_irq *irq)
 723 {
 724         napi_disable(&irq->napi);
 725         irq_set_affinity_notifier(irq->irq, NULL);
 726         irq_update_affinity_hint(irq->irq, NULL);
 727         free_irq(irq->irq, irq);
 728         irq->state = FUN_IRQ_INIT;
 729 }
 730
 731 static void fun_disable_irqs(struct net_device *dev)
 732 {
 733         struct funeth_priv *fp = netdev_priv(dev);
 734         struct fun_irq *p;
 735         unsigned long idx;
 736
 737         xa_for_each(&fp->irqs, idx, p)
 738                 if (p->state == FUN_IRQ_ENABLED)
 739                         fun_disable_one_irq(p);
 740 }
 741
 742 static void fun_down(struct net_device *dev, struct fun_qset *qset)
 743 {
 744         struct funeth_priv *fp = netdev_priv(dev);
 745
 746         /* If we don't have queues the data path is already down.
 747          * Note netif_running(dev) may be true.
 748          */
 749         if (!rcu_access_pointer(fp->rxqs))
 750                 return;
 751
 752         /* It is also down if the queues aren't on the device. */
 753         if (fp->txqs[0]->init_state >= FUN_QSTATE_INIT_FULL) {
 754                 netif_info(fp, ifdown, dev,
 755                            "Tearing down data path on device\n");
 756                 fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_DISABLE, 0);
 757
 758                 netif_carrier_off(dev);
 759                 netif_tx_disable(dev);
 760
 761                 fun_destroy_rss(fp);
 762                 fun_res_destroy(fp->fdev, FUN_ADMIN_OP_VI, 0, dev->dev_port);
 763                 fun_disable_irqs(dev);
 764         }
 765
 766         fun_free_rings(dev, qset);
 767 }
 768
 769 static int fun_up(struct net_device *dev, struct fun_qset *qset)
 770 {
 771         static const int port_keys[] = {
 772                 FUN_ADMIN_PORT_KEY_STATS_DMA_LOW,
 773                 FUN_ADMIN_PORT_KEY_STATS_DMA_HIGH,
 774                 FUN_ADMIN_PORT_KEY_ENABLE
 775         };
 776
 777         struct funeth_priv *fp = netdev_priv(dev);
 778         u64 vals[] = {
 779                 lower_32_bits(fp->stats_dma_addr),
 780                 upper_32_bits(fp->stats_dma_addr),
 781                 FUN_PORT_FLAG_ENABLE_NOTIFY
 782         };
 783         int err;
 784
 785         netif_info(fp, ifup, dev, "Setting up data path on device\n");
 786
 787         if (qset->rxqs[0]->init_state < FUN_QSTATE_INIT_FULL) {
 788                 err = fun_advance_ring_state(dev, qset);
 789                 if (err)
 790                         return err;
 791         }
 792
 793         err = fun_vi_create(fp);
 794         if (err)
 795                 goto free_queues;
 796
 797         fp->txqs = qset->txqs;
 798         rcu_assign_pointer(fp->rxqs, qset->rxqs);
 799         rcu_assign_pointer(fp->xdpqs, qset->xdpqs);
 800
 801         err = fun_enable_irqs(dev);
 802         if (err)
 803                 goto destroy_vi;
 804
 805         if (fp->rss_cfg) {
 806                 err = fun_config_rss(dev, fp->hash_algo, fp->rss_key,
 807                                      fp->indir_table, FUN_ADMIN_SUBOP_CREATE);
 808         } else {
 809                 /* The non-RSS case has only 1 queue. */
 810                 err = fun_bind(fp->fdev, FUN_ADMIN_BIND_TYPE_VI, dev->dev_port,
 811                                FUN_ADMIN_BIND_TYPE_EPCQ,
 812                                qset->rxqs[0]->hw_cqid);
 813         }
 814         if (err)
 815                 goto disable_irqs;
 816
 817         err = fun_port_write_cmds(fp, 3, port_keys, vals);
 818         if (err)
 819                 goto free_rss;
 820
 821         netif_tx_start_all_queues(dev);
 822         return 0;
 823
 824 free_rss:
 825         fun_destroy_rss(fp);
 826 disable_irqs:
 827         fun_disable_irqs(dev);
 828 destroy_vi:
 829         fun_res_destroy(fp->fdev, FUN_ADMIN_OP_VI, 0, dev->dev_port);
 830 free_queues:
 831         fun_free_rings(dev, qset);
 832         return err;
 833 }
 834
 835 static int funeth_open(struct net_device *netdev)
 836 {
 837         struct funeth_priv *fp = netdev_priv(netdev);
 838         struct fun_qset qset = {
 839                 .nrxqs = netdev->real_num_rx_queues,
 840                 .ntxqs = netdev->real_num_tx_queues,
 841                 .nxdpqs = fp->num_xdpqs,
 842                 .cq_depth = fp->cq_depth,
 843                 .rq_depth = fp->rq_depth,
 844                 .sq_depth = fp->sq_depth,
 845                 .state = FUN_QSTATE_INIT_FULL,
 846         };
 847         int rc;
 848
 849         rc = fun_alloc_rings(netdev, &qset);
 850         if (rc)
 851                 return rc;
 852
 853         rc = fun_up(netdev, &qset);
 854         if (rc) {
 855                 qset.state = FUN_QSTATE_DESTROYED;
 856                 fun_free_rings(netdev, &qset);
 857         }
 858
 859         return rc;
 860 }
 861
 862 static int funeth_close(struct net_device *netdev)
 863 {
 864         struct fun_qset qset = { .state = FUN_QSTATE_DESTROYED };
 865
 866         fun_down(netdev, &qset);
 867         return 0;
 868 }
 869
 870 static void fun_get_stats64(struct net_device *netdev,
 871                             struct rtnl_link_stats64 *stats)
 872 {
 873         struct funeth_priv *fp = netdev_priv(netdev);
 874         struct funeth_txq **xdpqs;
 875         struct funeth_rxq **rxqs;
 876         unsigned int i, start;
 877
 878         stats->tx_packets = fp->tx_packets;
 879         stats->tx_bytes   = fp->tx_bytes;
 880         stats->tx_dropped = fp->tx_dropped;
 881
 882         stats->rx_packets = fp->rx_packets;
 883         stats->rx_bytes   = fp->rx_bytes;
 884         stats->rx_dropped = fp->rx_dropped;
 885
 886         rcu_read_lock();
 887         rxqs = rcu_dereference(fp->rxqs);
 888         if (!rxqs)
 889                 goto unlock;
 890
 891         for (i = 0; i < netdev->real_num_tx_queues; i++) {
 892                 struct funeth_txq_stats txs;
 893
 894                 FUN_QSTAT_READ(fp->txqs[i], start, txs);
 895                 stats->tx_packets += txs.tx_pkts;
 896                 stats->tx_bytes   += txs.tx_bytes;
 897                 stats->tx_dropped += txs.tx_map_err;
 898         }
 899
 900         for (i = 0; i < netdev->real_num_rx_queues; i++) {
 901                 struct funeth_rxq_stats rxs;
 902
 903                 FUN_QSTAT_READ(rxqs[i], start, rxs);
 904                 stats->rx_packets += rxs.rx_pkts;
 905                 stats->rx_bytes   += rxs.rx_bytes;
 906                 stats->rx_dropped += rxs.rx_map_err + rxs.rx_mem_drops;
 907         }
 908
 909         xdpqs = rcu_dereference(fp->xdpqs);
 910         if (!xdpqs)
 911                 goto unlock;
 912
 913         for (i = 0; i < fp->num_xdpqs; i++) {
 914                 struct funeth_txq_stats txs;
 915
 916                 FUN_QSTAT_READ(xdpqs[i], start, txs);
 917                 stats->tx_packets += txs.tx_pkts;
 918                 stats->tx_bytes   += txs.tx_bytes;
 919         }
 920 unlock:
 921         rcu_read_unlock();
 922 }
 923
 924 static int fun_change_mtu(struct net_device *netdev, int new_mtu)
 925 {
 926         struct funeth_priv *fp = netdev_priv(netdev);
 927         int rc;
 928
 929         rc = fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_MTU, new_mtu);
 930         if (!rc)
 931                 netdev->mtu = new_mtu;
 932         return rc;
 933 }
 934
 935 static int fun_set_macaddr(struct net_device *netdev, void *addr)
 936 {
 937         struct funeth_priv *fp = netdev_priv(netdev);
 938         struct sockaddr *saddr = addr;
 939         int rc;
 940
 941         if (!is_valid_ether_addr(saddr->sa_data))
 942                 return -EADDRNOTAVAIL;
 943
 944         if (ether_addr_equal(netdev->dev_addr, saddr->sa_data))
 945                 return 0;
 946
 947         rc = fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_MACADDR,
 948                                 ether_addr_to_u64(saddr->sa_data));
 949         if (!rc)
 950                 eth_hw_addr_set(netdev, saddr->sa_data);
 951         return rc;
 952 }
 953
 954 static int fun_get_port_attributes(struct net_device *netdev)
 955 {
 956         static const int keys[] = {
 957                 FUN_ADMIN_PORT_KEY_MACADDR, FUN_ADMIN_PORT_KEY_CAPABILITIES,
 958                 FUN_ADMIN_PORT_KEY_ADVERT, FUN_ADMIN_PORT_KEY_MTU
 959         };
 960         static const int phys_keys[] = {
 961                 FUN_ADMIN_PORT_KEY_LANE_ATTRS,
 962         };
 963
 964         struct funeth_priv *fp = netdev_priv(netdev);
 965         u64 data[ARRAY_SIZE(keys)];
 966         u8 mac[ETH_ALEN];
 967         int i, rc;
 968
 969         rc = fun_port_read_cmds(fp, ARRAY_SIZE(keys), keys, data);
 970         if (rc)
 971                 return rc;
 972
 973         for (i = 0; i < ARRAY_SIZE(keys); i++) {
 974                 switch (keys[i]) {
 975                 case FUN_ADMIN_PORT_KEY_MACADDR:
 976                         u64_to_ether_addr(data[i], mac);
 977                         if (is_zero_ether_addr(mac)) {
 978                                 eth_hw_addr_random(netdev);
 979                         } else if (is_valid_ether_addr(mac)) {
 980                                 eth_hw_addr_set(netdev, mac);
 981                         } else {
 982                                 netdev_err(netdev,
 983                                            "device provided a bad MAC address %pM\n",
 984                                            mac);
 985                                 return -EINVAL;
 986                         }
 987                         break;
 988
 989                 case FUN_ADMIN_PORT_KEY_CAPABILITIES:
 990                         fp->port_caps = data[i];
 991                         break;
 992
 993                 case FUN_ADMIN_PORT_KEY_ADVERT:
 994                         fp->advertising = data[i];
 995                         break;
 996
 997                 case FUN_ADMIN_PORT_KEY_MTU:
 998                         netdev->mtu = data[i];
 999                         break;
1000                 }
1001         }
1002
1003         if (!(fp->port_caps & FUN_PORT_CAP_VPORT)) {
1004                 rc = fun_port_read_cmds(fp, ARRAY_SIZE(phys_keys), phys_keys,
1005                                         data);
1006                 if (rc)
1007                         return rc;
1008
1009                 fp->lane_attrs = data[0];
1010         }
1011
1012         if (netdev->addr_assign_type == NET_ADDR_RANDOM)
1013                 return fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_MACADDR,
1014                                           ether_addr_to_u64(netdev->dev_addr));
1015         return 0;
1016 }
1017
1018 static int fun_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
1019 {
1020         const struct funeth_priv *fp = netdev_priv(dev);
1021
1022         return copy_to_user(ifr->ifr_data, &fp->hwtstamp_cfg,
1023                             sizeof(fp->hwtstamp_cfg)) ? -EFAULT : 0;
1024 }
1025
1026 static int fun_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
1027 {
1028         struct funeth_priv *fp = netdev_priv(dev);
1029         struct hwtstamp_config cfg;
1030
1031         if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1032                 return -EFAULT;
1033
1034         /* no TX HW timestamps */
1035         cfg.tx_type = HWTSTAMP_TX_OFF;
1036
1037         switch (cfg.rx_filter) {
1038         case HWTSTAMP_FILTER_NONE:
1039                 break;
1040         case HWTSTAMP_FILTER_ALL:
1041         case HWTSTAMP_FILTER_SOME:
1042         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1043         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1044         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1045         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1046         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1047         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1048         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1049         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1050         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1051         case HWTSTAMP_FILTER_PTP_V2_EVENT:
1052         case HWTSTAMP_FILTER_PTP_V2_SYNC:
1053         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1054         case HWTSTAMP_FILTER_NTP_ALL:
1055                 cfg.rx_filter = HWTSTAMP_FILTER_ALL;
1056                 break;
1057         default:
1058                 return -ERANGE;
1059         }
1060
1061         fp->hwtstamp_cfg = cfg;
1062         return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1063 }
1064
1065 static int fun_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
1066 {
1067         switch (cmd) {
1068         case SIOCSHWTSTAMP:
1069                 return fun_hwtstamp_set(dev, ifr);
1070         case SIOCGHWTSTAMP:
1071                 return fun_hwtstamp_get(dev, ifr);
1072         default:
1073                 return -EOPNOTSUPP;
1074         }
1075 }
1076
1077 /* Prepare the queues for XDP. */
1078 static int fun_enter_xdp(struct net_device *dev, struct bpf_prog *prog)
1079 {
1080         struct funeth_priv *fp = netdev_priv(dev);
1081         unsigned int i, nqs = num_online_cpus();
1082         struct funeth_txq **xdpqs;
1083         struct funeth_rxq **rxqs;
1084         int err;
1085
1086         xdpqs = alloc_xdpqs(dev, nqs, fp->sq_depth, 0, FUN_QSTATE_INIT_FULL);
1087         if (IS_ERR(xdpqs))
1088                 return PTR_ERR(xdpqs);
1089
1090         rxqs = rtnl_dereference(fp->rxqs);
1091         for (i = 0; i < dev->real_num_rx_queues; i++) {
1092                 err = fun_rxq_set_bpf(rxqs[i], prog);
1093                 if (err)
1094                         goto out;
1095         }
1096
1097         fp->num_xdpqs = nqs;
1098         rcu_assign_pointer(fp->xdpqs, xdpqs);
1099         return 0;
1100 out:
1101         while (i--)
1102                 fun_rxq_set_bpf(rxqs[i], NULL);
1103
1104         free_xdpqs(xdpqs, nqs, 0, FUN_QSTATE_DESTROYED);
1105         return err;
1106 }
1107
1108 /* Set the queues for non-XDP operation. */
1109 static void fun_end_xdp(struct net_device *dev)
1110 {
1111         struct funeth_priv *fp = netdev_priv(dev);
1112         struct funeth_txq **xdpqs;
1113         struct funeth_rxq **rxqs;
1114         unsigned int i;
1115
1116         xdpqs = rtnl_dereference(fp->xdpqs);
1117         rcu_assign_pointer(fp->xdpqs, NULL);
1118         synchronize_net();
1119         /* at this point both Rx and Tx XDP processing has ended */
1120
1121         free_xdpqs(xdpqs, fp->num_xdpqs, 0, FUN_QSTATE_DESTROYED);
1122         fp->num_xdpqs = 0;
1123
1124         rxqs = rtnl_dereference(fp->rxqs);
1125         for (i = 0; i < dev->real_num_rx_queues; i++)
1126                 fun_rxq_set_bpf(rxqs[i], NULL);
1127 }
1128
1129 #define XDP_MAX_MTU \
1130         (PAGE_SIZE - FUN_XDP_HEADROOM - VLAN_ETH_HLEN - FUN_RX_TAILROOM)
1131
1132 static int fun_xdp_setup(struct net_device *dev, struct netdev_bpf *xdp)
1133 {
1134         struct bpf_prog *old_prog, *prog = xdp->prog;
1135         struct funeth_priv *fp = netdev_priv(dev);
1136         int i, err;
1137
1138         /* XDP uses at most one buffer */
1139         if (prog && dev->mtu > XDP_MAX_MTU) {
1140                 netdev_err(dev, "device MTU %u too large for XDP\n", dev->mtu);
1141                 NL_SET_ERR_MSG_MOD(xdp->extack,
1142                                    "Device MTU too large for XDP");
1143                 return -EINVAL;
1144         }
1145
1146         if (!netif_running(dev)) {
1147                 fp->num_xdpqs = prog ? num_online_cpus() : 0;
1148         } else if (prog && !fp->xdp_prog) {
1149                 err = fun_enter_xdp(dev, prog);
1150                 if (err) {
1151                         NL_SET_ERR_MSG_MOD(xdp->extack,
1152                                            "Failed to set queues for XDP.");
1153                         return err;
1154                 }
1155         } else if (!prog && fp->xdp_prog) {
1156                 fun_end_xdp(dev);
1157         } else {
1158                 struct funeth_rxq **rxqs = rtnl_dereference(fp->rxqs);
1159
1160                 for (i = 0; i < dev->real_num_rx_queues; i++)
1161                         WRITE_ONCE(rxqs[i]->xdp_prog, prog);
1162         }
1163
1164         dev->max_mtu = prog ? XDP_MAX_MTU : FUN_MAX_MTU;
1165         old_prog = xchg(&fp->xdp_prog, prog);
1166         if (old_prog)
1167                 bpf_prog_put(old_prog);
1168
1169         return 0;
1170 }
1171
1172 static int fun_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1173 {
1174         switch (xdp->command) {
1175         case XDP_SETUP_PROG:
1176                 return fun_xdp_setup(dev, xdp);
1177         default:
1178                 return -EINVAL;
1179         }
1180 }
1181
1182 static struct devlink_port *fun_get_devlink_port(struct net_device *netdev)
1183 {
1184         struct funeth_priv *fp = netdev_priv(netdev);
1185
1186         return &fp->dl_port;
1187 }
1188
1189 static int fun_init_vports(struct fun_ethdev *ed, unsigned int n)
1190 {
1191         if (ed->num_vports)
1192                 return -EINVAL;
1193
1194         ed->vport_info = kvcalloc(n, sizeof(*ed->vport_info), GFP_KERNEL);
1195         if (!ed->vport_info)
1196                 return -ENOMEM;
1197         ed->num_vports = n;
1198         return 0;
1199 }
1200
1201 static void fun_free_vports(struct fun_ethdev *ed)
1202 {
1203         kvfree(ed->vport_info);
1204         ed->vport_info = NULL;
1205         ed->num_vports = 0;
1206 }
1207
1208 static struct fun_vport_info *fun_get_vport(struct fun_ethdev *ed,
1209                                             unsigned int vport)
1210 {
1211         if (!ed->vport_info || vport >= ed->num_vports)
1212                 return NULL;
1213
1214         return ed->vport_info + vport;
1215 }
1216
1217 static int fun_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
1218 {
1219         struct funeth_priv *fp = netdev_priv(dev);
1220         struct fun_adi_param mac_param = {};
1221         struct fun_dev *fdev = fp->fdev;
1222         struct fun_ethdev *ed = to_fun_ethdev(fdev);
1223         struct fun_vport_info *vi;
1224         int rc = -EINVAL;
1225
1226         if (is_multicast_ether_addr(mac))
1227                 return -EINVAL;
1228
1229         mutex_lock(&ed->state_mutex);
1230         vi = fun_get_vport(ed, vf);
1231         if (!vi)
1232                 goto unlock;
1233
1234         mac_param.u.mac = FUN_ADI_MAC_INIT(ether_addr_to_u64(mac));
1235         rc = fun_adi_write(fdev, FUN_ADMIN_ADI_ATTR_MACADDR, vf + 1,
1236                            &mac_param);
1237         if (!rc)
1238                 ether_addr_copy(vi->mac, mac);
1239 unlock:
1240         mutex_unlock(&ed->state_mutex);
1241         return rc;
1242 }
1243
1244 static int fun_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
1245                            __be16 vlan_proto)
1246 {
1247         struct funeth_priv *fp = netdev_priv(dev);
1248         struct fun_adi_param vlan_param = {};
1249         struct fun_dev *fdev = fp->fdev;
1250         struct fun_ethdev *ed = to_fun_ethdev(fdev);
1251         struct fun_vport_info *vi;
1252         int rc = -EINVAL;
1253
1254         if (vlan > 4095 || qos > 7)
1255                 return -EINVAL;
1256         if (vlan_proto && vlan_proto != htons(ETH_P_8021Q) &&
1257             vlan_proto != htons(ETH_P_8021AD))
1258                 return -EINVAL;
1259
1260         mutex_lock(&ed->state_mutex);
1261         vi = fun_get_vport(ed, vf);
1262         if (!vi)
1263                 goto unlock;
1264
1265         vlan_param.u.vlan = FUN_ADI_VLAN_INIT(be16_to_cpu(vlan_proto),
1266                                               ((u16)qos << VLAN_PRIO_SHIFT) | vlan);
1267         rc = fun_adi_write(fdev, FUN_ADMIN_ADI_ATTR_VLAN, vf + 1, &vlan_param);
1268         if (!rc) {
1269                 vi->vlan = vlan;
1270                 vi->qos = qos;
1271                 vi->vlan_proto = vlan_proto;
1272         }
1273 unlock:
1274         mutex_unlock(&ed->state_mutex);
1275         return rc;
1276 }
1277
1278 static int fun_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
1279                            int max_tx_rate)
1280 {
1281         struct funeth_priv *fp = netdev_priv(dev);
1282         struct fun_adi_param rate_param = {};
1283         struct fun_dev *fdev = fp->fdev;
1284         struct fun_ethdev *ed = to_fun_ethdev(fdev);
1285         struct fun_vport_info *vi;
1286         int rc = -EINVAL;
1287
1288         if (min_tx_rate)
1289                 return -EINVAL;
1290
1291         mutex_lock(&ed->state_mutex);
1292         vi = fun_get_vport(ed, vf);
1293         if (!vi)
1294                 goto unlock;
1295
1296         rate_param.u.rate = FUN_ADI_RATE_INIT(max_tx_rate);
1297         rc = fun_adi_write(fdev, FUN_ADMIN_ADI_ATTR_RATE, vf + 1, &rate_param);
1298         if (!rc)
1299                 vi->max_rate = max_tx_rate;
1300 unlock:
1301         mutex_unlock(&ed->state_mutex);
1302         return rc;
1303 }
1304
1305 static int fun_get_vf_config(struct net_device *dev, int vf,
1306                              struct ifla_vf_info *ivi)
1307 {
1308         struct funeth_priv *fp = netdev_priv(dev);
1309         struct fun_ethdev *ed = to_fun_ethdev(fp->fdev);
1310         const struct fun_vport_info *vi;
1311
1312         mutex_lock(&ed->state_mutex);
1313         vi = fun_get_vport(ed, vf);
1314         if (!vi)
1315                 goto unlock;
1316
1317         memset(ivi, 0, sizeof(*ivi));
1318         ivi->vf = vf;
1319         ether_addr_copy(ivi->mac, vi->mac);
1320         ivi->vlan = vi->vlan;
1321         ivi->qos = vi->qos;
1322         ivi->vlan_proto = vi->vlan_proto;
1323         ivi->max_tx_rate = vi->max_rate;
1324         ivi->spoofchk = vi->spoofchk;
1325 unlock:
1326         mutex_unlock(&ed->state_mutex);
1327         return vi ? 0 : -EINVAL;
1328 }
1329
1330 static void fun_uninit(struct net_device *dev)
1331 {
1332         struct funeth_priv *fp = netdev_priv(dev);
1333
1334         fun_prune_queue_irqs(dev);
1335         xa_destroy(&fp->irqs);
1336 }
1337
1338 static const struct net_device_ops fun_netdev_ops = {
1339         .ndo_open               = funeth_open,
1340         .ndo_stop               = funeth_close,
1341         .ndo_start_xmit         = fun_start_xmit,
1342         .ndo_get_stats64        = fun_get_stats64,
1343         .ndo_change_mtu         = fun_change_mtu,
1344         .ndo_set_mac_address    = fun_set_macaddr,
1345         .ndo_validate_addr      = eth_validate_addr,
1346         .ndo_eth_ioctl          = fun_ioctl,
1347         .ndo_uninit             = fun_uninit,
1348         .ndo_bpf                = fun_xdp,
1349         .ndo_xdp_xmit           = fun_xdp_xmit_frames,
1350         .ndo_set_vf_mac         = fun_set_vf_mac,
1351         .ndo_set_vf_vlan        = fun_set_vf_vlan,
1352         .ndo_set_vf_rate        = fun_set_vf_rate,
1353         .ndo_get_vf_config      = fun_get_vf_config,
1354         .ndo_get_devlink_port   = fun_get_devlink_port,
1355 };
1356
1357 #define GSO_ENCAP_FLAGS (NETIF_F_GSO_GRE | NETIF_F_GSO_IPXIP4 | \
1358                          NETIF_F_GSO_IPXIP6 | NETIF_F_GSO_UDP_TUNNEL | \
1359                          NETIF_F_GSO_UDP_TUNNEL_CSUM)
1360 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
1361 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_HW_CSUM | TSO_FLAGS | \
1362                    GSO_ENCAP_FLAGS | NETIF_F_HIGHDMA)
1363
1364 static void fun_dflt_rss_indir(struct funeth_priv *fp, unsigned int nrx)
1365 {
1366         unsigned int i;
1367
1368         for (i = 0; i < fp->indir_table_nentries; i++)
1369                 fp->indir_table[i] = ethtool_rxfh_indir_default(i, nrx);
1370 }
1371
1372 /* Reset the RSS indirection table to equal distribution across the current
1373  * number of Rx queues. Called at init time and whenever the number of Rx
1374  * queues changes subsequently. Note that this may also resize the indirection
1375  * table.
1376  */
1377 static void fun_reset_rss_indir(struct net_device *dev, unsigned int nrx)
1378 {
1379         struct funeth_priv *fp = netdev_priv(dev);
1380
1381         if (!fp->rss_cfg)
1382                 return;
1383
1384         /* Set the table size to the max possible that allows an equal number
1385          * of occurrences of each CQ.
1386          */
1387         fp->indir_table_nentries = rounddown(FUN_ETH_RSS_MAX_INDIR_ENT, nrx);
1388         fun_dflt_rss_indir(fp, nrx);
1389 }
1390
1391 /* Update the RSS LUT to contain only queues in [0, nrx). Normally this will
1392  * update the LUT to an equal distribution among nrx queues, If @only_if_needed
1393  * is set the LUT is left unchanged if it already does not reference any queues
1394  * >= nrx.
1395  */
1396 static int fun_rss_set_qnum(struct net_device *dev, unsigned int nrx,
1397                             bool only_if_needed)
1398 {
1399         struct funeth_priv *fp = netdev_priv(dev);
1400         u32 old_lut[FUN_ETH_RSS_MAX_INDIR_ENT];
1401         unsigned int i, oldsz;
1402         int err;
1403
1404         if (!fp->rss_cfg)
1405                 return 0;
1406
1407         if (only_if_needed) {
1408                 for (i = 0; i < fp->indir_table_nentries; i++)
1409                         if (fp->indir_table[i] >= nrx)
1410                                 break;
1411
1412                 if (i >= fp->indir_table_nentries)
1413                         return 0;
1414         }
1415
1416         memcpy(old_lut, fp->indir_table, sizeof(old_lut));
1417         oldsz = fp->indir_table_nentries;
1418         fun_reset_rss_indir(dev, nrx);
1419
1420         err = fun_config_rss(dev, fp->hash_algo, fp->rss_key,
1421                              fp->indir_table, FUN_ADMIN_SUBOP_MODIFY);
1422         if (!err)
1423                 return 0;
1424
1425         memcpy(fp->indir_table, old_lut, sizeof(old_lut));
1426         fp->indir_table_nentries = oldsz;
1427         return err;
1428 }
1429
1430 /* Allocate the DMA area for the RSS configuration commands to the device, and
1431  * initialize the hash, hash key, indirection table size and its entries to
1432  * their defaults. The indirection table defaults to equal distribution across
1433  * the Rx queues.
1434  */
1435 static int fun_init_rss(struct net_device *dev)
1436 {
1437         struct funeth_priv *fp = netdev_priv(dev);
1438         size_t size = sizeof(fp->rss_key) + sizeof(fp->indir_table);
1439
1440         fp->rss_hw_id = FUN_HCI_ID_INVALID;
1441         if (!(fp->port_caps & FUN_PORT_CAP_OFFLOADS))
1442                 return 0;
1443
1444         fp->rss_cfg = dma_alloc_coherent(&fp->pdev->dev, size,
1445                                          &fp->rss_dma_addr, GFP_KERNEL);
1446         if (!fp->rss_cfg)
1447                 return -ENOMEM;
1448
1449         fp->hash_algo = FUN_ETH_RSS_ALG_TOEPLITZ;
1450         netdev_rss_key_fill(fp->rss_key, sizeof(fp->rss_key));
1451         fun_reset_rss_indir(dev, dev->real_num_rx_queues);
1452         return 0;
1453 }
1454
1455 static void fun_free_rss(struct funeth_priv *fp)
1456 {
1457         if (fp->rss_cfg) {
1458                 dma_free_coherent(&fp->pdev->dev,
1459                                   sizeof(fp->rss_key) + sizeof(fp->indir_table),
1460                                   fp->rss_cfg, fp->rss_dma_addr);
1461                 fp->rss_cfg = NULL;
1462         }
1463 }
1464
1465 void fun_set_ring_count(struct net_device *netdev, unsigned int ntx,
1466                         unsigned int nrx)
1467 {
1468         netif_set_real_num_tx_queues(netdev, ntx);
1469         if (nrx != netdev->real_num_rx_queues) {
1470                 netif_set_real_num_rx_queues(netdev, nrx);
1471                 fun_reset_rss_indir(netdev, nrx);
1472         }
1473 }
1474
1475 static int fun_init_stats_area(struct funeth_priv *fp)
1476 {
1477         unsigned int nstats;
1478
1479         if (!(fp->port_caps & FUN_PORT_CAP_STATS))
1480                 return 0;
1481
1482         nstats = PORT_MAC_RX_STATS_MAX + PORT_MAC_TX_STATS_MAX +
1483                  PORT_MAC_FEC_STATS_MAX;
1484
1485         fp->stats = dma_alloc_coherent(&fp->pdev->dev, nstats * sizeof(u64),
1486                                        &fp->stats_dma_addr, GFP_KERNEL);
1487         if (!fp->stats)
1488                 return -ENOMEM;
1489         return 0;
1490 }
1491
1492 static void fun_free_stats_area(struct funeth_priv *fp)
1493 {
1494         unsigned int nstats;
1495
1496         if (fp->stats) {
1497                 nstats = PORT_MAC_RX_STATS_MAX + PORT_MAC_TX_STATS_MAX;
1498                 dma_free_coherent(&fp->pdev->dev, nstats * sizeof(u64),
1499                                   fp->stats, fp->stats_dma_addr);
1500                 fp->stats = NULL;
1501         }
1502 }
1503
1504 static int fun_dl_port_register(struct net_device *netdev)
1505 {
1506         struct funeth_priv *fp = netdev_priv(netdev);
1507         struct devlink *dl = priv_to_devlink(fp->fdev);
1508         struct devlink_port_attrs attrs = {};
1509         unsigned int idx;
1510
1511         if (fp->port_caps & FUN_PORT_CAP_VPORT) {
1512                 attrs.flavour = DEVLINK_PORT_FLAVOUR_VIRTUAL;
1513                 idx = fp->lport;
1514         } else {
1515                 idx = netdev->dev_port;
1516                 attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
1517                 attrs.lanes = fp->lane_attrs & 7;
1518                 if (fp->lane_attrs & FUN_PORT_LANE_SPLIT) {
1519                         attrs.split = 1;
1520                         attrs.phys.port_number = fp->lport & ~3;
1521                         attrs.phys.split_subport_number = fp->lport & 3;
1522                 } else {
1523                         attrs.phys.port_number = fp->lport;
1524                 }
1525         }
1526
1527         devlink_port_attrs_set(&fp->dl_port, &attrs);
1528
1529         return devlink_port_register(dl, &fp->dl_port, idx);
1530 }
1531
1532 /* Determine the max Tx/Rx queues for a port. */
1533 static int fun_max_qs(struct fun_ethdev *ed, unsigned int *ntx,
1534                       unsigned int *nrx)
1535 {
1536         int neth;
1537
1538         if (ed->num_ports > 1 || is_kdump_kernel()) {
1539                 *ntx = 1;
1540                 *nrx = 1;
1541                 return 0;
1542         }
1543
1544         neth = fun_get_res_count(&ed->fdev, FUN_ADMIN_OP_ETH);
1545         if (neth < 0)
1546                 return neth;
1547
1548         /* We determine the max number of queues based on the CPU
1549          * cores, device interrupts and queues, RSS size, and device Tx flows.
1550          *
1551          * - At least 1 Rx and 1 Tx queues.
1552          * - At most 1 Rx/Tx queue per core.
1553          * - Each Rx/Tx queue needs 1 SQ.
1554          */
1555         *ntx = min(ed->nsqs_per_port - 1, num_online_cpus());
1556         *nrx = *ntx;
1557         if (*ntx > neth)
1558                 *ntx = neth;
1559         if (*nrx > FUN_ETH_RSS_MAX_INDIR_ENT)
1560                 *nrx = FUN_ETH_RSS_MAX_INDIR_ENT;
1561         return 0;
1562 }
1563
1564 static void fun_queue_defaults(struct net_device *dev, unsigned int nsqs)
1565 {
1566         unsigned int ntx, nrx;
1567
1568         ntx = min(dev->num_tx_queues, FUN_DFLT_QUEUES);
1569         nrx = min(dev->num_rx_queues, FUN_DFLT_QUEUES);
1570         if (ntx <= nrx) {
1571                 ntx = min(ntx, nsqs / 2);
1572                 nrx = min(nrx, nsqs - ntx);
1573         } else {
1574                 nrx = min(nrx, nsqs / 2);
1575                 ntx = min(ntx, nsqs - nrx);
1576         }
1577
1578         netif_set_real_num_tx_queues(dev, ntx);
1579         netif_set_real_num_rx_queues(dev, nrx);
1580 }
1581
1582 /* Replace the existing Rx/Tx/XDP queues with equal number of queues with
1583  * different settings, e.g. depth. This is a disruptive replacement that
1584  * temporarily shuts down the data path and should be limited to changes that
1585  * can't be applied to live queues. The old queues are always discarded.
1586  */
1587 int fun_replace_queues(struct net_device *dev, struct fun_qset *newqs,
1588                        struct netlink_ext_ack *extack)
1589 {
1590         struct fun_qset oldqs = { .state = FUN_QSTATE_DESTROYED };
1591         struct funeth_priv *fp = netdev_priv(dev);
1592         int err;
1593
1594         newqs->nrxqs = dev->real_num_rx_queues;
1595         newqs->ntxqs = dev->real_num_tx_queues;
1596         newqs->nxdpqs = fp->num_xdpqs;
1597         newqs->state = FUN_QSTATE_INIT_SW;
1598         err = fun_alloc_rings(dev, newqs);
1599         if (err) {
1600                 NL_SET_ERR_MSG_MOD(extack,
1601                                    "Unable to allocate memory for new queues, keeping current settings");
1602                 return err;
1603         }
1604
1605         fun_down(dev, &oldqs);
1606
1607         err = fun_up(dev, newqs);
1608         if (!err)
1609                 return 0;
1610
1611         /* The new queues couldn't be installed. We do not retry the old queues
1612          * as they are the same to the device as the new queues and would
1613          * similarly fail.
1614          */
1615         newqs->state = FUN_QSTATE_DESTROYED;
1616         fun_free_rings(dev, newqs);
1617         NL_SET_ERR_MSG_MOD(extack, "Unable to restore the data path with the new queues.");
1618         return err;
1619 }
1620
1621 /* Change the number of Rx/Tx queues of a device while it is up. This is done
1622  * by incrementally adding/removing queues to meet the new requirements while
1623  * handling ongoing traffic.
1624  */
1625 int fun_change_num_queues(struct net_device *dev, unsigned int ntx,
1626                           unsigned int nrx)
1627 {
1628         unsigned int keep_tx = min(dev->real_num_tx_queues, ntx);
1629         unsigned int keep_rx = min(dev->real_num_rx_queues, nrx);
1630         struct funeth_priv *fp = netdev_priv(dev);
1631         struct fun_qset oldqs = {
1632                 .rxqs = rtnl_dereference(fp->rxqs),
1633                 .txqs = fp->txqs,
1634                 .nrxqs = dev->real_num_rx_queues,
1635                 .ntxqs = dev->real_num_tx_queues,
1636                 .rxq_start = keep_rx,
1637                 .txq_start = keep_tx,
1638                 .state = FUN_QSTATE_DESTROYED
1639         };
1640         struct fun_qset newqs = {
1641                 .nrxqs = nrx,
1642                 .ntxqs = ntx,
1643                 .rxq_start = keep_rx,
1644                 .txq_start = keep_tx,
1645                 .cq_depth = fp->cq_depth,
1646                 .rq_depth = fp->rq_depth,
1647                 .sq_depth = fp->sq_depth,
1648                 .state = FUN_QSTATE_INIT_FULL
1649         };
1650         int i, err;
1651
1652         err = fun_alloc_rings(dev, &newqs);
1653         if (err)
1654                 goto free_irqs;
1655
1656         err = fun_enable_irqs(dev); /* of any newly added queues */
1657         if (err)
1658                 goto free_rings;
1659
1660         /* copy the queues we are keeping to the new set */
1661         memcpy(newqs.rxqs, oldqs.rxqs, keep_rx * sizeof(*oldqs.rxqs));
1662         memcpy(newqs.txqs, fp->txqs, keep_tx * sizeof(*fp->txqs));
1663
1664         if (nrx < dev->real_num_rx_queues) {
1665                 err = fun_rss_set_qnum(dev, nrx, true);
1666                 if (err)
1667                         goto disable_tx_irqs;
1668
1669                 for (i = nrx; i < dev->real_num_rx_queues; i++)
1670                         fun_disable_one_irq(container_of(oldqs.rxqs[i]->napi,
1671                                                          struct fun_irq, napi));
1672
1673                 netif_set_real_num_rx_queues(dev, nrx);
1674         }
1675
1676         if (ntx < dev->real_num_tx_queues)
1677                 netif_set_real_num_tx_queues(dev, ntx);
1678
1679         rcu_assign_pointer(fp->rxqs, newqs.rxqs);
1680         fp->txqs = newqs.txqs;
1681         synchronize_net();
1682
1683         if (ntx > dev->real_num_tx_queues)
1684                 netif_set_real_num_tx_queues(dev, ntx);
1685
1686         if (nrx > dev->real_num_rx_queues) {
1687                 netif_set_real_num_rx_queues(dev, nrx);
1688                 fun_rss_set_qnum(dev, nrx, false);
1689         }
1690
1691         /* disable interrupts of any excess Tx queues */
1692         for (i = keep_tx; i < oldqs.ntxqs; i++)
1693                 fun_disable_one_irq(oldqs.txqs[i]->irq);
1694
1695         fun_free_rings(dev, &oldqs);
1696         fun_prune_queue_irqs(dev);
1697         return 0;
1698
1699 disable_tx_irqs:
1700         for (i = oldqs.ntxqs; i < ntx; i++)
1701                 fun_disable_one_irq(newqs.txqs[i]->irq);
1702 free_rings:
1703         newqs.state = FUN_QSTATE_DESTROYED;
1704         fun_free_rings(dev, &newqs);
1705 free_irqs:
1706         fun_prune_queue_irqs(dev);
1707         return err;
1708 }
1709
1710 static int fun_create_netdev(struct fun_ethdev *ed, unsigned int portid)
1711 {
1712         struct fun_dev *fdev = &ed->fdev;
1713         struct net_device *netdev;
1714         struct funeth_priv *fp;
1715         unsigned int ntx, nrx;
1716         int rc;
1717
1718         rc = fun_max_qs(ed, &ntx, &nrx);
1719         if (rc)
1720                 return rc;
1721
1722         netdev = alloc_etherdev_mqs(sizeof(*fp), ntx, nrx);
1723         if (!netdev) {
1724                 rc = -ENOMEM;
1725                 goto done;
1726         }
1727
1728         netdev->dev_port = portid;
1729         fun_queue_defaults(netdev, ed->nsqs_per_port);
1730
1731         fp = netdev_priv(netdev);
1732         fp->fdev = fdev;
1733         fp->pdev = to_pci_dev(fdev->dev);
1734         fp->netdev = netdev;
1735         xa_init(&fp->irqs);
1736         fp->rx_irq_ofst = ntx;
1737         seqcount_init(&fp->link_seq);
1738
1739         fp->lport = INVALID_LPORT;
1740         rc = fun_port_create(netdev);
1741         if (rc)
1742                 goto free_netdev;
1743
1744         /* bind port to admin CQ for async events */
1745         rc = fun_bind(fdev, FUN_ADMIN_BIND_TYPE_PORT, portid,
1746                       FUN_ADMIN_BIND_TYPE_EPCQ, 0);
1747         if (rc)
1748                 goto destroy_port;
1749
1750         rc = fun_get_port_attributes(netdev);
1751         if (rc)
1752                 goto destroy_port;
1753
1754         rc = fun_init_rss(netdev);
1755         if (rc)
1756                 goto destroy_port;
1757
1758         rc = fun_init_stats_area(fp);
1759         if (rc)
1760                 goto free_rss;
1761
1762         SET_NETDEV_DEV(netdev, fdev->dev);
1763         netdev->netdev_ops = &fun_netdev_ops;
1764
1765         netdev->hw_features = NETIF_F_SG | NETIF_F_RXHASH | NETIF_F_RXCSUM;
1766         if (fp->port_caps & FUN_PORT_CAP_OFFLOADS)
1767                 netdev->hw_features |= NETIF_F_HW_CSUM | TSO_FLAGS;
1768         if (fp->port_caps & FUN_PORT_CAP_ENCAP_OFFLOADS)
1769                 netdev->hw_features |= GSO_ENCAP_FLAGS;
1770
1771         netdev->features |= netdev->hw_features | NETIF_F_HIGHDMA;
1772         netdev->vlan_features = netdev->features & VLAN_FEAT;
1773         netdev->mpls_features = netdev->vlan_features;
1774         netdev->hw_enc_features = netdev->hw_features;
1775
1776         netdev->min_mtu = ETH_MIN_MTU;
1777         netdev->max_mtu = FUN_MAX_MTU;
1778
1779         fun_set_ethtool_ops(netdev);
1780
1781         /* configurable parameters */
1782         fp->sq_depth = min(SQ_DEPTH, fdev->q_depth);
1783         fp->cq_depth = min(CQ_DEPTH, fdev->q_depth);
1784         fp->rq_depth = min_t(unsigned int, RQ_DEPTH, fdev->q_depth);
1785         fp->rx_coal_usec  = CQ_INTCOAL_USEC;
1786         fp->rx_coal_count = CQ_INTCOAL_NPKT;
1787         fp->tx_coal_usec  = SQ_INTCOAL_USEC;
1788         fp->tx_coal_count = SQ_INTCOAL_NPKT;
1789         fp->cq_irq_db = FUN_IRQ_CQ_DB(fp->rx_coal_usec, fp->rx_coal_count);
1790
1791         rc = fun_dl_port_register(netdev);
1792         if (rc)
1793                 goto free_stats;
1794
1795         fp->ktls_id = FUN_HCI_ID_INVALID;
1796         fun_ktls_init(netdev);            /* optional, failure OK */
1797
1798         netif_carrier_off(netdev);
1799         ed->netdevs[portid] = netdev;
1800         rc = register_netdev(netdev);
1801         if (rc)
1802                 goto unreg_devlink;
1803
1804         if (fp->dl_port.devlink)
1805                 devlink_port_type_eth_set(&fp->dl_port, netdev);
1806
1807         return 0;
1808
1809 unreg_devlink:
1810         ed->netdevs[portid] = NULL;
1811         fun_ktls_cleanup(fp);
1812         if (fp->dl_port.devlink)
1813                 devlink_port_unregister(&fp->dl_port);
1814 free_stats:
1815         fun_free_stats_area(fp);
1816 free_rss:
1817         fun_free_rss(fp);
1818 destroy_port:
1819         fun_port_destroy(netdev);
1820 free_netdev:
1821         free_netdev(netdev);
1822 done:
1823         dev_err(fdev->dev, "couldn't allocate port %u, error %d", portid, rc);
1824         return rc;
1825 }
1826
1827 static void fun_destroy_netdev(struct net_device *netdev)
1828 {
1829         struct funeth_priv *fp;
1830
1831         fp = netdev_priv(netdev);
1832         if (fp->dl_port.devlink) {
1833                 devlink_port_type_clear(&fp->dl_port);
1834                 devlink_port_unregister(&fp->dl_port);
1835         }
1836         unregister_netdev(netdev);
1837         fun_ktls_cleanup(fp);
1838         fun_free_stats_area(fp);
1839         fun_free_rss(fp);
1840         fun_port_destroy(netdev);
1841         free_netdev(netdev);
1842 }
1843
1844 static int fun_create_ports(struct fun_ethdev *ed, unsigned int nports)
1845 {
1846         struct fun_dev *fd = &ed->fdev;
1847         int i, rc;
1848
1849         /* The admin queue takes 1 IRQ and 2 SQs. */
1850         ed->nsqs_per_port = min(fd->num_irqs - 1,
1851                                 fd->kern_end_qid - 2) / nports;
1852         if (ed->nsqs_per_port < 2) {
1853                 dev_err(fd->dev, "Too few SQs for %u ports", nports);
1854                 return -EINVAL;
1855         }
1856
1857         ed->netdevs = kcalloc(nports, sizeof(*ed->netdevs), GFP_KERNEL);
1858         if (!ed->netdevs)
1859                 return -ENOMEM;
1860
1861         ed->num_ports = nports;
1862         for (i = 0; i < nports; i++) {
1863                 rc = fun_create_netdev(ed, i);
1864                 if (rc)
1865                         goto free_netdevs;
1866         }
1867
1868         return 0;
1869
1870 free_netdevs:
1871         while (i)
1872                 fun_destroy_netdev(ed->netdevs[--i]);
1873         kfree(ed->netdevs);
1874         ed->netdevs = NULL;
1875         ed->num_ports = 0;
1876         return rc;
1877 }
1878
1879 static void fun_destroy_ports(struct fun_ethdev *ed)
1880 {
1881         unsigned int i;
1882
1883         for (i = 0; i < ed->num_ports; i++)
1884                 fun_destroy_netdev(ed->netdevs[i]);
1885
1886         kfree(ed->netdevs);
1887         ed->netdevs = NULL;
1888         ed->num_ports = 0;
1889 }
1890
1891 static void fun_update_link_state(const struct fun_ethdev *ed,
1892                                   const struct fun_admin_port_notif *notif)
1893 {
1894         unsigned int port_idx = be16_to_cpu(notif->id);
1895         struct net_device *netdev;
1896         struct funeth_priv *fp;
1897
1898         if (port_idx >= ed->num_ports)
1899                 return;
1900
1901         netdev = ed->netdevs[port_idx];
1902         fp = netdev_priv(netdev);
1903
1904         write_seqcount_begin(&fp->link_seq);
1905         fp->link_speed = be32_to_cpu(notif->speed) * 10;  /* 10 Mbps->Mbps */
1906         fp->active_fc = notif->flow_ctrl;
1907         fp->active_fec = notif->fec;
1908         fp->xcvr_type = notif->xcvr_type;
1909         fp->link_down_reason = notif->link_down_reason;
1910         fp->lp_advertising = be64_to_cpu(notif->lp_advertising);
1911
1912         if ((notif->link_state | notif->missed_events) & FUN_PORT_FLAG_MAC_DOWN)
1913                 netif_carrier_off(netdev);
1914         if (notif->link_state & FUN_PORT_FLAG_MAC_UP)
1915                 netif_carrier_on(netdev);
1916
1917         write_seqcount_end(&fp->link_seq);
1918         fun_report_link(netdev);
1919 }
1920
1921 /* handler for async events delivered through the admin CQ */
1922 static void fun_event_cb(struct fun_dev *fdev, void *entry)
1923 {
1924         u8 op = ((struct fun_admin_rsp_common *)entry)->op;
1925
1926         if (op == FUN_ADMIN_OP_PORT) {
1927                 const struct fun_admin_port_notif *rsp = entry;
1928
1929                 if (rsp->subop == FUN_ADMIN_SUBOP_NOTIFY) {
1930                         fun_update_link_state(to_fun_ethdev(fdev), rsp);
1931                 } else if (rsp->subop == FUN_ADMIN_SUBOP_RES_COUNT) {
1932                         const struct fun_admin_res_count_rsp *r = entry;
1933
1934                         if (r->count.data)
1935                                 set_bit(FUN_SERV_RES_CHANGE, &fdev->service_flags);
1936                         else
1937                                 set_bit(FUN_SERV_DEL_PORTS, &fdev->service_flags);
1938                         fun_serv_sched(fdev);
1939                 } else {
1940                         dev_info(fdev->dev, "adminq event unexpected op %u subop %u",
1941                                  op, rsp->subop);
1942                 }
1943         } else {
1944                 dev_info(fdev->dev, "adminq event unexpected op %u", op);
1945         }
1946 }
1947
1948 /* handler for pending work managed by the service task */
1949 static void fun_service_cb(struct fun_dev *fdev)
1950 {
1951         struct fun_ethdev *ed = to_fun_ethdev(fdev);
1952         int rc;
1953
1954         if (test_and_clear_bit(FUN_SERV_DEL_PORTS, &fdev->service_flags))
1955                 fun_destroy_ports(ed);
1956
1957         if (!test_and_clear_bit(FUN_SERV_RES_CHANGE, &fdev->service_flags))
1958                 return;
1959
1960         rc = fun_get_res_count(fdev, FUN_ADMIN_OP_PORT);
1961         if (rc < 0 || rc == ed->num_ports)
1962                 return;
1963
1964         if (ed->num_ports)
1965                 fun_destroy_ports(ed);
1966         if (rc)
1967                 fun_create_ports(ed, rc);
1968 }
1969
1970 static int funeth_sriov_configure(struct pci_dev *pdev, int nvfs)
1971 {
1972         struct fun_dev *fdev = pci_get_drvdata(pdev);
1973         struct fun_ethdev *ed = to_fun_ethdev(fdev);
1974         int rc;
1975
1976         if (nvfs == 0) {
1977                 if (pci_vfs_assigned(pdev)) {
1978                         dev_warn(&pdev->dev,
1979                                  "Cannot disable SR-IOV while VFs are assigned\n");
1980                         return -EPERM;
1981                 }
1982
1983                 mutex_lock(&ed->state_mutex);
1984                 fun_free_vports(ed);
1985                 mutex_unlock(&ed->state_mutex);
1986                 pci_disable_sriov(pdev);
1987                 return 0;
1988         }
1989
1990         rc = pci_enable_sriov(pdev, nvfs);
1991         if (rc)
1992                 return rc;
1993
1994         mutex_lock(&ed->state_mutex);
1995         rc = fun_init_vports(ed, nvfs);
1996         mutex_unlock(&ed->state_mutex);
1997         if (rc) {
1998                 pci_disable_sriov(pdev);
1999                 return rc;
2000         }
2001
2002         return nvfs;
2003 }
2004
2005 static int funeth_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2006 {
2007         struct fun_dev_params aqreq = {
2008                 .cqe_size_log2 = ilog2(ADMIN_CQE_SIZE),
2009                 .sqe_size_log2 = ilog2(ADMIN_SQE_SIZE),
2010                 .cq_depth      = ADMIN_CQ_DEPTH,
2011                 .sq_depth      = ADMIN_SQ_DEPTH,
2012                 .rq_depth      = ADMIN_RQ_DEPTH,
2013                 .min_msix      = 2,              /* 1 Rx + 1 Tx */
2014                 .event_cb      = fun_event_cb,
2015                 .serv_cb       = fun_service_cb,
2016         };
2017         struct devlink *devlink;
2018         struct fun_ethdev *ed;
2019         struct fun_dev *fdev;
2020         int rc;
2021
2022         devlink = fun_devlink_alloc(&pdev->dev);
2023         if (!devlink) {
2024                 dev_err(&pdev->dev, "devlink alloc failed\n");
2025                 return -ENOMEM;
2026         }
2027
2028         ed = devlink_priv(devlink);
2029         mutex_init(&ed->state_mutex);
2030
2031         fdev = &ed->fdev;
2032         rc = fun_dev_enable(fdev, pdev, &aqreq, KBUILD_MODNAME);
2033         if (rc)
2034                 goto free_devlink;
2035
2036         rc = fun_get_res_count(fdev, FUN_ADMIN_OP_PORT);
2037         if (rc > 0)
2038                 rc = fun_create_ports(ed, rc);
2039         if (rc < 0)
2040                 goto disable_dev;
2041
2042         fun_serv_restart(fdev);
2043         fun_devlink_register(devlink);
2044         return 0;
2045
2046 disable_dev:
2047         fun_dev_disable(fdev);
2048 free_devlink:
2049         mutex_destroy(&ed->state_mutex);
2050         fun_devlink_free(devlink);
2051         return rc;
2052 }
2053
2054 static void funeth_remove(struct pci_dev *pdev)
2055 {
2056         struct fun_dev *fdev = pci_get_drvdata(pdev);
2057         struct devlink *devlink;
2058         struct fun_ethdev *ed;
2059
2060         ed = to_fun_ethdev(fdev);
2061         devlink = priv_to_devlink(ed);
2062         fun_devlink_unregister(devlink);
2063
2064 #ifdef CONFIG_PCI_IOV
2065         funeth_sriov_configure(pdev, 0);
2066 #endif
2067
2068         fun_serv_stop(fdev);
2069         fun_destroy_ports(ed);
2070         fun_dev_disable(fdev);
2071         mutex_destroy(&ed->state_mutex);
2072
2073         fun_devlink_free(devlink);
2074 }
2075
2076 static struct pci_driver funeth_driver = {
2077         .name            = KBUILD_MODNAME,
2078         .id_table        = funeth_id_table,
2079         .probe           = funeth_probe,
2080         .remove          = funeth_remove,
2081         .shutdown        = funeth_remove,
2082         .sriov_configure = funeth_sriov_configure,
2083 };
2084
2085 module_pci_driver(funeth_driver);
2086
2087 MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");
2088 MODULE_DESCRIPTION("Fungible Ethernet Network Driver");
2089 MODULE_LICENSE("Dual BSD/GPL");
2090 MODULE_DEVICE_TABLE(pci, funeth_id_table);